wasmtime 22.0.0 → 23.0.2

data/ext/cargo-vendor/wasmtime-23.0.2/src/runtime/vm/gc.rs

@@ -0,0 +1,244 @@
+#[cfg(feature = "gc")]
+mod enabled;
+#[cfg(feature = "gc")]
+pub use enabled::*;
+
+#[cfg(not(feature = "gc"))]
+mod disabled;
+#[cfg(not(feature = "gc"))]
+pub use disabled::*;
+
+mod gc_ref;
+mod gc_runtime;
+mod host_data;
+mod i31;
+
+pub use gc_ref::*;
+pub use gc_runtime::*;
+pub use host_data::*;
+pub use i31::*;
+
+use crate::prelude::*;
+use crate::runtime::vm::GcHeapAllocationIndex;
+use core::ptr;
+use core::{any::Any, num::NonZeroUsize};
+use wasmtime_environ::StackMap;
+
+/// Used by the runtime to lookup information about a module given a
+/// program counter value.
+pub trait ModuleInfoLookup {
+    /// Lookup the module information from a program counter value.
+    fn lookup(&self, pc: usize) -> Option<&dyn ModuleInfo>;
+}
+
+/// Used by the runtime to query module information.
+pub trait ModuleInfo {
+    /// Lookup the stack map at a program counter value.
+    fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap>;
+}
+
+/// GC-related data that is one-to-one with a `wasmtime::Store`.
+///
+/// Contains everything we need to do collections, invoke barriers, etc...
+///
+/// In general, exposes a very similar interface to `GcHeap`, but fills in some
+/// of the context arguments for callers (such as the `ExternRefHostDataTable`)
+/// since they are all stored together inside `GcStore`.
+pub struct GcStore {
+    /// This GC heap's allocation index (primarily used for integrating with the
+    /// pooling allocator).
+    pub allocation_index: GcHeapAllocationIndex,
+
+    /// The actual GC heap.
+    pub gc_heap: Box<dyn GcHeap>,
+
+    /// The `externref` host data table for this GC heap.
+    pub host_data_table: ExternRefHostDataTable,
+}
+
+impl GcStore {
+    /// Create a new `GcStore`.
+    pub fn new(allocation_index: GcHeapAllocationIndex, gc_heap: Box<dyn GcHeap>) -> Self {
+        let host_data_table = ExternRefHostDataTable::default();
+        Self {
+            allocation_index,
+            gc_heap,
+            host_data_table,
+        }
+    }
+
+    /// Perform garbage collection within this heap.
+    pub fn gc(&mut self, roots: GcRootsIter<'_>) {
+        let mut collection = self.gc_heap.gc(roots, &mut self.host_data_table);
+        collection.collect();
+    }
+
+    /// Asynchronously perform garbage collection within this heap.
+    #[cfg(feature = "async")]
+    pub async fn gc_async(&mut self, roots: GcRootsIter<'_>) {
+        let collection = self.gc_heap.gc(roots, &mut self.host_data_table);
+        collect_async(collection).await;
+    }
+
+    /// Clone a GC reference, calling GC write barriers as necessary.
+    pub fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
+        if gc_ref.is_i31() {
+            gc_ref.unchecked_copy()
+        } else {
+            self.gc_heap.clone_gc_ref(gc_ref)
+        }
+    }
+
+    /// Write the `source` GC reference into the `destination` slot, performing
+    /// write barriers as necessary.
+    pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
+        // If neither the source nor destination actually point to a GC object
+        // (that is, they are both either null or `i31ref`s) then we can skip
+        // the GC barrier.
+        if destination.as_ref().map_or(true, |d| d.is_i31())
+            && source.as_ref().map_or(true, |s| s.is_i31())
+        {
+            *destination = source.map(|s| s.unchecked_copy());
+            return;
+        }
+
+        self.gc_heap
+            .write_gc_ref(&mut self.host_data_table, destination, source);
+    }
+
+    /// Drop the given GC reference, performing drop barriers as necessary.
+    pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
+        if !gc_ref.is_i31() {
+            self.gc_heap.drop_gc_ref(&mut self.host_data_table, gc_ref);
+        }
+    }
+
+    /// Hook to call whenever a GC reference is about to be exposed to Wasm.
+    pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) {
+        if !gc_ref.is_i31() {
+            self.gc_heap.expose_gc_ref_to_wasm(gc_ref);
+        }
+    }
+
+    /// Allocate a new `externref`.
+    ///
+    /// Returns:
+    ///
+    /// * `Ok(Ok(_))`: Successfully allocated the `externref`.
+    ///
+    /// * `Ok(Err(value))`: Failed to allocate the `externref`, but doing a GC
+    ///   and then trying again may succeed. Returns the given `value` as the
+    ///   error payload.
+    ///
+    /// * `Err(_)`: Unrecoverable allocation failure.
+    pub fn alloc_externref(
+        &mut self,
+        value: Box<dyn Any + Send + Sync>,
+    ) -> Result<Result<VMExternRef, Box<dyn Any + Send + Sync>>> {
+        let host_data_id = self.host_data_table.alloc(value);
+        match self.gc_heap.alloc_externref(host_data_id)? {
+            Some(x) => Ok(Ok(x)),
+            None => Ok(Err(self.host_data_table.dealloc(host_data_id))),
+        }
+    }
+
+    /// Get a shared borrow of the given `externref`'s host data.
+    ///
+    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
+    /// associated with a different heap is memory safe but will lead to general
+    /// incorrectness such as panics and wrong results.
+    pub fn externref_host_data(&self, externref: &VMExternRef) -> &(dyn Any + Send + Sync) {
+        let host_data_id = self.gc_heap.externref_host_data(externref);
+        self.host_data_table.get(host_data_id)
+    }
+
+    /// Get a mutable borrow of the given `externref`'s host data.
+    ///
+    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
+    /// associated with a different heap is memory safe but will lead to general
+    /// incorrectness such as panics and wrong results.
+    pub fn externref_host_data_mut(
+        &mut self,
+        externref: &VMExternRef,
+    ) -> &mut (dyn Any + Send + Sync) {
+        let host_data_id = self.gc_heap.externref_host_data(externref);
+        self.host_data_table.get_mut(host_data_id)
+    }
+}
+
+/// Get a no-op GC heap for when GC is disabled (either statically at compile
+/// time or dynamically due to it being turned off in the `wasmtime::Config`).
+pub fn disabled_gc_heap() -> Box<dyn GcHeap> {
+    return Box::new(DisabledGcHeap);
+
+    struct DisabledGcHeap;
+
+    unsafe impl GcHeap for DisabledGcHeap {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+        fn as_any_mut(&mut self) -> &mut dyn Any {
+            self
+        }
+        fn enter_no_gc_scope(&mut self) {}
+        fn exit_no_gc_scope(&mut self) {}
+        fn header(&self, _gc_ref: &VMGcRef) -> &VMGcHeader {
+            unreachable!()
+        }
+        fn clone_gc_ref(&mut self, _gc_ref: &VMGcRef) -> VMGcRef {
+            unreachable!()
+        }
+        fn write_gc_ref(
+            &mut self,
+            _host_data_table: &mut ExternRefHostDataTable,
+            _destination: &mut Option<VMGcRef>,
+            _source: Option<&VMGcRef>,
+        ) {
+            unreachable!()
+        }
+        fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) {
+            unreachable!()
+        }
+        fn need_gc_before_entering_wasm(&self, _num_gc_refs: NonZeroUsize) -> bool {
+            unreachable!()
+        }
+        fn alloc_externref(
+            &mut self,
+            _host_data: ExternRefHostDataId,
+        ) -> Result<Option<VMExternRef>> {
+            bail!(
+                "GC support disabled either in the `Config` or at compile time \
+                 because the `gc` cargo feature was not enabled"
+            )
+        }
+        fn externref_host_data(&self, _externref: &VMExternRef) -> ExternRefHostDataId {
+            unreachable!()
+        }
+        fn gc<'a>(
+            &'a mut self,
+            _roots: GcRootsIter<'a>,
+            _host_data_table: &'a mut ExternRefHostDataTable,
+        ) -> Box<dyn GarbageCollection<'a> + 'a> {
+            return Box::new(NoGc);
+
+            struct NoGc;
+
+            impl<'a> GarbageCollection<'a> for NoGc {
+                fn collect_increment(&mut self) -> GcProgress {
+                    GcProgress::Complete
+                }
+            }
+        }
+        unsafe fn vmctx_gc_heap_base(&self) -> *mut u8 {
+            ptr::null_mut()
+        }
+        unsafe fn vmctx_gc_heap_bound(&self) -> usize {
+            0
+        }
+        unsafe fn vmctx_gc_heap_data(&self) -> *mut u8 {
+            ptr::null_mut()
+        }
+        #[cfg(feature = "pooling-allocator")]
+        fn reset(&mut self) {}
+    }
+}

data/ext/cargo-vendor/wasmtime-23.0.2/src/runtime/vm/instance/allocator/on_demand.rs

@@ -0,0 +1,219 @@
+use super::{
+    InstanceAllocationRequest, InstanceAllocatorImpl, MemoryAllocationIndex, TableAllocationIndex,
+};
+use crate::prelude::*;
+use crate::runtime::vm::instance::RuntimeMemoryCreator;
+use crate::runtime::vm::memory::{DefaultMemoryCreator, Memory};
+use crate::runtime::vm::mpk::ProtectionKey;
+use crate::runtime::vm::table::Table;
+use crate::runtime::vm::CompiledModuleId;
+use alloc::sync::Arc;
+use wasmtime_environ::{
+    DefinedMemoryIndex, DefinedTableIndex, HostPtr, MemoryPlan, Module, TablePlan, VMOffsets,
+};
+
+#[cfg(feature = "gc")]
+use crate::runtime::vm::{GcHeap, GcHeapAllocationIndex, GcRuntime};
+
+#[cfg(feature = "async")]
+use wasmtime_fiber::RuntimeFiberStackCreator;
+
+#[cfg(feature = "component-model")]
+use wasmtime_environ::{
+    component::{Component, VMComponentOffsets},
+    StaticModuleIndex,
+};
+
+/// Represents the on-demand instance allocator.
+#[derive(Clone)]
+pub struct OnDemandInstanceAllocator {
+    mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>,
+    #[cfg(feature = "async")]
+    stack_creator: Option<Arc<dyn RuntimeFiberStackCreator>>,
+    #[cfg(feature = "async")]
+    stack_size: usize,
+}
+
+impl OnDemandInstanceAllocator {
+    /// Creates a new on-demand instance allocator.
+    pub fn new(mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>, stack_size: usize) -> Self {
+        let _ = stack_size; // suppress warnings when async feature is disabled.
+        Self {
+            mem_creator,
+            #[cfg(feature = "async")]
+            stack_creator: None,
+            #[cfg(feature = "async")]
+            stack_size,
+        }
+    }
+
+    /// Set the stack creator.
+    #[cfg(feature = "async")]
+    pub fn set_stack_creator(&mut self, stack_creator: Arc<dyn RuntimeFiberStackCreator>) {
+        self.stack_creator = Some(stack_creator);
+    }
+}
+
+impl Default for OnDemandInstanceAllocator {
+    fn default() -> Self {
+        Self {
+            mem_creator: None,
+            #[cfg(feature = "async")]
+            stack_creator: None,
+            #[cfg(feature = "async")]
+            stack_size: 0,
+        }
+    }
+}
+
+unsafe impl InstanceAllocatorImpl for OnDemandInstanceAllocator {
+    #[cfg(feature = "component-model")]
+    fn validate_component_impl<'a>(
+        &self,
+        _component: &Component,
+        _offsets: &VMComponentOffsets<HostPtr>,
+        _get_module: &'a dyn Fn(StaticModuleIndex) -> &'a Module,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn validate_module_impl(&self, _module: &Module, _offsets: &VMOffsets<HostPtr>) -> Result<()> {
+        Ok(())
+    }
+
+    fn increment_component_instance_count(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn decrement_component_instance_count(&self) {}
+
+    fn increment_core_instance_count(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn decrement_core_instance_count(&self) {}
+
+    unsafe fn allocate_memory(
+        &self,
+        request: &mut InstanceAllocationRequest,
+        memory_plan: &MemoryPlan,
+        memory_index: DefinedMemoryIndex,
+    ) -> Result<(MemoryAllocationIndex, Memory)> {
+        let creator = self
+            .mem_creator
+            .as_deref()
+            .unwrap_or_else(|| &DefaultMemoryCreator);
+        let image = request.runtime_info.memory_image(memory_index)?;
+        let allocation_index = MemoryAllocationIndex::default();
+        let memory = Memory::new_dynamic(
+            memory_plan,
+            creator,
+            request
+                .store
+                .get()
+                .expect("if module has memory plans, store is not empty"),
+            image,
+        )?;
+        Ok((allocation_index, memory))
+    }
+
+    unsafe fn deallocate_memory(
+        &self,
+        _memory_index: DefinedMemoryIndex,
+        allocation_index: MemoryAllocationIndex,
+        _memory: Memory,
+    ) {
+        debug_assert_eq!(allocation_index, MemoryAllocationIndex::default());
+        // Normal destructors do all the necessary clean up.
+    }
+
+    unsafe fn allocate_table(
+        &self,
+        request: &mut InstanceAllocationRequest,
+        table_plan: &TablePlan,
+        _table_index: DefinedTableIndex,
+    ) -> Result<(TableAllocationIndex, Table)> {
+        let allocation_index = TableAllocationIndex::default();
+        let table = Table::new_dynamic(
+            table_plan,
+            request
+                .store
+                .get()
+                .expect("if module has table plans, store is not empty"),
+        )?;
+        Ok((allocation_index, table))
+    }
+
+    unsafe fn deallocate_table(
+        &self,
+        _table_index: DefinedTableIndex,
+        allocation_index: TableAllocationIndex,
+        _table: Table,
+    ) {
+        debug_assert_eq!(allocation_index, TableAllocationIndex::default());
+        // Normal destructors do all the necessary clean up.
+    }
+
+    #[cfg(feature = "async")]
+    fn allocate_fiber_stack(&self) -> Result<wasmtime_fiber::FiberStack> {
+        if self.stack_size == 0 {
+            anyhow::bail!("fiber stacks are not supported by the allocator")
+        }
+        let stack = match &self.stack_creator {
+            Some(stack_creator) => {
+                let stack = stack_creator.new_stack(self.stack_size)?;
+                wasmtime_fiber::FiberStack::from_custom(stack)
+            }
+            None => wasmtime_fiber::FiberStack::new(self.stack_size),
+        }?;
+        Ok(stack)
+    }
+
+    #[cfg(feature = "async")]
+    unsafe fn deallocate_fiber_stack(&self, stack: wasmtime_fiber::FiberStack) {
+        // The on-demand allocator has no further bookkeeping for fiber stacks
+        // beyond dropping them.
+        let _ = stack;
+    }
+
+    fn purge_module(&self, _: CompiledModuleId) {}
+
+    fn next_available_pkey(&self) -> Option<ProtectionKey> {
+        // The on-demand allocator cannot use protection keys--it requires
+        // back-to-back allocation of memory slots that this allocator cannot
+        // guarantee.
+        None
+    }
+
+    fn restrict_to_pkey(&self, _: ProtectionKey) {
+        // The on-demand allocator cannot use protection keys; an on-demand
+        // allocator will never hand out protection keys to the stores its
+        // engine creates.
+        unreachable!()
+    }
+
+    fn allow_all_pkeys(&self) {
+        // The on-demand allocator cannot use protection keys; an on-demand
+        // allocator will never hand out protection keys to the stores its
+        // engine creates.
+        unreachable!()
+    }
+
+    #[cfg(feature = "gc")]
+    fn allocate_gc_heap(
+        &self,
+        gc_runtime: &dyn GcRuntime,
+    ) -> Result<(GcHeapAllocationIndex, Box<dyn GcHeap>)> {
+        Ok((GcHeapAllocationIndex::default(), gc_runtime.new_gc_heap()?))
+    }
+
+    #[cfg(feature = "gc")]
+    fn deallocate_gc_heap(
+        &self,
+        allocation_index: GcHeapAllocationIndex,
+        gc_heap: Box<dyn crate::runtime::vm::GcHeap>,
+    ) {
+        debug_assert_eq!(allocation_index, GcHeapAllocationIndex::default());
+        drop(gc_heap);
+    }
+}

data/ext/cargo-vendor/wasmtime-23.0.2/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs

@@ -0,0 +1,199 @@
+//! A queue for batching decommits together.
+//!
+//! We don't immediately decommit a Wasm table/memory/stack/etc... eagerly, but
+//! instead batch them up to be decommitted together. This module implements
+//! that queuing and batching.
+//!
+//! Even when batching is "disabled" we still use this queue. Batching is
+//! disabled by specifying a batch size of one, in which case, this queue will
+//! immediately get flushed every time we push onto it.
+
+use super::PoolingInstanceAllocator;
+use crate::vm::{MemoryAllocationIndex, MemoryImageSlot, Table, TableAllocationIndex};
+use smallvec::SmallVec;
+
+#[cfg(feature = "async")]
+use wasmtime_fiber::FiberStack;
+
+#[cfg(unix)]
+#[allow(non_camel_case_types)]
+type iovec = libc::iovec;
+
+#[cfg(not(unix))]
+#[allow(non_camel_case_types)]
+struct iovec {
+    iov_base: *mut libc::c_void,
+    iov_len: libc::size_t,
+}
+
+#[repr(transparent)]
+struct IoVec(iovec);
+
+unsafe impl Send for IoVec {}
+unsafe impl Sync for IoVec {}
+
+impl std::fmt::Debug for IoVec {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.debug_struct("IoVec")
+            .field("base", &self.0.iov_base)
+            .field("len", &self.0.iov_len)
+            .finish()
+    }
+}
+
+#[cfg(feature = "async")]
+struct SendSyncStack(FiberStack);
+#[cfg(feature = "async")]
+unsafe impl Send for SendSyncStack {}
+#[cfg(feature = "async")]
+unsafe impl Sync for SendSyncStack {}
+
+#[derive(Default)]
+pub struct DecommitQueue {
+    raw: SmallVec<[IoVec; 2]>,
+    memories: SmallVec<[(MemoryAllocationIndex, MemoryImageSlot); 1]>,
+    tables: SmallVec<[(TableAllocationIndex, Table); 1]>,
+    #[cfg(feature = "async")]
+    stacks: SmallVec<[SendSyncStack; 1]>,
+    //
+    // TODO: GC heaps are not well-integrated with the pooling allocator
+    // yet. Once we better integrate them, we should start (optionally) zeroing
+    // them, and batching that up here.
+    //
+    // #[cfg(feature = "gc")]
+    // pub gc_heaps: SmallVec<[(GcHeapAllocationIndex, Box<dyn GcHeap>); 1]>,
+}
+
+impl std::fmt::Debug for DecommitQueue {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DecommitQueue")
+            .field("raw", &self.raw)
+            .finish_non_exhaustive()
+    }
+}
+
+impl DecommitQueue {
+    /// Append another queue to this queue.
+    pub fn append(
+        &mut self,
+        Self {
+            raw,
+            memories,
+            tables,
+            #[cfg(feature = "async")]
+            stacks,
+        }: &mut Self,
+    ) {
+        self.raw.append(raw);
+        self.memories.append(memories);
+        self.tables.append(tables);
+        #[cfg(feature = "async")]
+        self.stacks.append(stacks);
+    }
+
+    /// How many raw memory regions are enqueued for decommit?
+    pub fn raw_len(&self) -> usize {
+        self.raw.len()
+    }
+
+    /// Enqueue a region of memory for decommit.
+    ///
+    /// It is the caller's responsibility to push the associated data via
+    /// `self.push_{memory,table,stack}` as appropriate.
+    ///
+    /// # Safety
+    ///
+    /// The enqueued memory regions must be safe to decommit when `flush` is
+    /// called (no other references, not in use, won't be otherwise unmapped,
+    /// etc...).
+    pub unsafe fn push_raw(&mut self, ptr: *mut u8, len: usize) {
+        self.raw.push(IoVec(iovec {
+            iov_base: ptr.cast(),
+            iov_len: len,
+        }));
+    }
+
+    /// Push a memory into the queue.
+    ///
+    /// # Safety
+    ///
+    /// This memory should not be in use, and its decommit regions must have
+    /// already been enqueued via `self.enqueue_raw`.
+    pub unsafe fn push_memory(
+        &mut self,
+        allocation_index: MemoryAllocationIndex,
+        image: MemoryImageSlot,
+    ) {
+        self.memories.push((allocation_index, image));
+    }
+
+    /// Push a table into the queue.
+    ///
+    /// # Safety
+    ///
+    /// This table should not be in use, and its decommit regions must have
+    /// already been enqueued via `self.enqueue_raw`.
+    pub unsafe fn push_table(&mut self, allocation_index: TableAllocationIndex, table: Table) {
+        self.tables.push((allocation_index, table));
+    }
+
+    /// Push a stack into the queue.
+    ///
+    /// # Safety
+    ///
+    /// This stack should not be in use, and its decommit regions must have
+    /// already been enqueued via `self.enqueue_raw`.
+    #[cfg(feature = "async")]
+    pub unsafe fn push_stack(&mut self, stack: FiberStack) {
+        self.stacks.push(SendSyncStack(stack));
+    }
+
+    fn decommit_all_raw(&mut self) {
+        for iovec in self.raw.drain(..) {
+            unsafe {
+                crate::vm::sys::vm::decommit_pages(iovec.0.iov_base.cast(), iovec.0.iov_len)
+                    .unwrap_or_else(|e| {
+                        panic!(
+                            "failed to decommit ptr={:#p}, len={:#x}: {e}",
+                            iovec.0.iov_base, iovec.0.iov_len
+                        )
+                    });
+            }
+        }
+    }
+
+    /// Flush this queue, decommitting all enqueued regions in batch.
+    ///
+    /// Returns `true` if we did any decommits and returned their entities to
+    /// the associated free lists; `false` if the queue was empty.
+    pub fn flush(mut self, pool: &PoolingInstanceAllocator) -> bool {
+        // First, do the raw decommit syscall(s).
+        self.decommit_all_raw();
+
+        // Second, restore the various entities to their associated pools' free
+        // lists. This is safe, and they are ready for reuse, now that their
+        // memory regions have been decommitted.
+        let mut deallocated_any = false;
+        for (allocation_index, image) in self.memories {
+            deallocated_any = true;
+            unsafe {
+                pool.memories.deallocate(allocation_index, image);
+            }
+        }
+        for (allocation_index, table) in self.tables {
+            deallocated_any = true;
+            unsafe {
+                pool.tables.deallocate(allocation_index, table);
+            }
+        }
+        #[cfg(feature = "async")]
+        for stack in self.stacks {
+            deallocated_any = true;
+            unsafe {
+                pool.stacks.deallocate(stack.0);
+            }
+        }
+
+        deallocated_any
+    }
+}