wasmtime 20.0.2 → 21.0.1

data/ext/cargo-vendor/wasmtime-21.0.1/src/runtime/vm/gc.rs

@@ -0,0 +1,245 @@
+#[cfg(feature = "gc")]
+mod enabled;
+#[cfg(feature = "gc")]
+pub use enabled::*;
+
+#[cfg(not(feature = "gc"))]
+mod disabled;
+#[cfg(not(feature = "gc"))]
+pub use disabled::*;
+
+mod gc_ref;
+mod gc_runtime;
+mod host_data;
+mod i31;
+
+pub use gc_ref::*;
+pub use gc_runtime::*;
+pub use host_data::*;
+pub use i31::*;
+
+use crate::prelude::*;
+use crate::runtime::vm::GcHeapAllocationIndex;
+use anyhow::{bail, Result};
+use core::ptr;
+use core::{any::Any, num::NonZeroUsize};
+use wasmtime_environ::StackMap;
+
+/// Used by the runtime to lookup information about a module given a
+/// program counter value.
+pub trait ModuleInfoLookup {
+    /// Lookup the module information from a program counter value.
+    fn lookup(&self, pc: usize) -> Option<&dyn ModuleInfo>;
+}
+
+/// Used by the runtime to query module information.
+pub trait ModuleInfo {
+    /// Lookup the stack map at a program counter value.
+    fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap>;
+}
+
+/// GC-related data that is one-to-one with a `wasmtime::Store`.
+///
+/// Contains everything we need to do collections, invoke barriers, etc...
+///
+/// In general, exposes a very similar interface to `GcHeap`, but fills in some
+/// of the context arguments for callers (such as the `ExternRefHostDataTable`)
+/// since they are all stored together inside `GcStore`.
+pub struct GcStore {
+    /// This GC heap's allocation index (primarily used for integrating with the
+    /// pooling allocator).
+    pub allocation_index: GcHeapAllocationIndex,
+
+    /// The actual GC heap.
+    pub gc_heap: Box<dyn GcHeap>,
+
+    /// The `externref` host data table for this GC heap.
+    pub host_data_table: ExternRefHostDataTable,
+}
+
+impl GcStore {
+    /// Create a new `GcStore`.
+    pub fn new(allocation_index: GcHeapAllocationIndex, gc_heap: Box<dyn GcHeap>) -> Self {
+        let host_data_table = ExternRefHostDataTable::default();
+        Self {
+            allocation_index,
+            gc_heap,
+            host_data_table,
+        }
+    }
+
+    /// Perform garbage collection within this heap.
+    pub fn gc(&mut self, roots: GcRootsIter<'_>) {
+        let mut collection = self.gc_heap.gc(roots, &mut self.host_data_table);
+        collection.collect();
+    }
+
+    /// Asynchronously perform garbage collection within this heap.
+    #[cfg(feature = "async")]
+    pub async fn gc_async(&mut self, roots: GcRootsIter<'_>) {
+        let collection = self.gc_heap.gc(roots, &mut self.host_data_table);
+        collect_async(collection).await;
+    }
+
+    /// Clone a GC reference, calling GC write barriers as necessary.
+    pub fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
+        if gc_ref.is_i31() {
+            gc_ref.unchecked_copy()
+        } else {
+            self.gc_heap.clone_gc_ref(gc_ref)
+        }
+    }
+
+    /// Write the `source` GC reference into the `destination` slot, performing
+    /// write barriers as necessary.
+    pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
+        // If neither the source nor destination actually point to a GC object
+        // (that is, they are both either null or `i31ref`s) then we can skip
+        // the GC barrier.
+        if destination.as_ref().map_or(true, |d| d.is_i31())
+            && source.as_ref().map_or(true, |s| s.is_i31())
+        {
+            *destination = source.map(|s| s.unchecked_copy());
+            return;
+        }
+
+        self.gc_heap
+            .write_gc_ref(&mut self.host_data_table, destination, source);
+    }
+
+    /// Drop the given GC reference, performing drop barriers as necessary.
+    pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
+        if !gc_ref.is_i31() {
+            self.gc_heap.drop_gc_ref(&mut self.host_data_table, gc_ref);
+        }
+    }
+
+    /// Hook to call whenever a GC reference is about to be exposed to Wasm.
+    pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) {
+        if !gc_ref.is_i31() {
+            self.gc_heap.expose_gc_ref_to_wasm(gc_ref);
+        }
+    }
+
+    /// Allocate a new `externref`.
+    ///
+    /// Returns:
+    ///
+    /// * `Ok(Ok(_))`: Successfully allocated the `externref`.
+    ///
+    /// * `Ok(Err(value))`: Failed to allocate the `externref`, but doing a GC
+    ///   and then trying again may succeed. Returns the given `value` as the
+    ///   error payload.
+    ///
+    /// * `Err(_)`: Unrecoverable allocation failure.
+    pub fn alloc_externref(
+        &mut self,
+        value: Box<dyn Any + Send + Sync>,
+    ) -> Result<Result<VMExternRef, Box<dyn Any + Send + Sync>>> {
+        let host_data_id = self.host_data_table.alloc(value);
+        match self.gc_heap.alloc_externref(host_data_id)? {
+            Some(x) => Ok(Ok(x)),
+            None => Ok(Err(self.host_data_table.dealloc(host_data_id))),
+        }
+    }
+
+    /// Get a shared borrow of the given `externref`'s host data.
+    ///
+    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
+    /// associated with a different heap is memory safe but will lead to general
+    /// incorrectness such as panics and wrong results.
+    pub fn externref_host_data(&self, externref: &VMExternRef) -> &(dyn Any + Send + Sync) {
+        let host_data_id = self.gc_heap.externref_host_data(externref);
+        self.host_data_table.get(host_data_id)
+    }
+
+    /// Get a mutable borrow of the given `externref`'s host data.
+    ///
+    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
+    /// associated with a different heap is memory safe but will lead to general
+    /// incorrectness such as panics and wrong results.
+    pub fn externref_host_data_mut(
+        &mut self,
+        externref: &VMExternRef,
+    ) -> &mut (dyn Any + Send + Sync) {
+        let host_data_id = self.gc_heap.externref_host_data(externref);
+        self.host_data_table.get_mut(host_data_id)
+    }
+}
+
+/// Get a no-op GC heap for when GC is disabled (either statically at compile
+/// time or dynamically due to it being turned off in the `wasmtime::Config`).
+pub fn disabled_gc_heap() -> Box<dyn GcHeap> {
+    return Box::new(DisabledGcHeap);
+
+    struct DisabledGcHeap;
+
+    unsafe impl GcHeap for DisabledGcHeap {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+        fn as_any_mut(&mut self) -> &mut dyn Any {
+            self
+        }
+        fn enter_no_gc_scope(&mut self) {}
+        fn exit_no_gc_scope(&mut self) {}
+        fn header(&self, _gc_ref: &VMGcRef) -> &VMGcHeader {
+            unreachable!()
+        }
+        fn clone_gc_ref(&mut self, _gc_ref: &VMGcRef) -> VMGcRef {
+            unreachable!()
+        }
+        fn write_gc_ref(
+            &mut self,
+            _host_data_table: &mut ExternRefHostDataTable,
+            _destination: &mut Option<VMGcRef>,
+            _source: Option<&VMGcRef>,
+        ) {
+            unreachable!()
+        }
+        fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) {
+            unreachable!()
+        }
+        fn need_gc_before_entering_wasm(&self, _num_gc_refs: NonZeroUsize) -> bool {
+            unreachable!()
+        }
+        fn alloc_externref(
+            &mut self,
+            _host_data: ExternRefHostDataId,
+        ) -> Result<Option<VMExternRef>> {
+            bail!(
+                "GC support disabled either in the `Config` or at compile time \
+                 because the `gc` cargo feature was not enabled"
+            )
+        }
+        fn externref_host_data(&self, _externref: &VMExternRef) -> ExternRefHostDataId {
+            unreachable!()
+        }
+        fn gc<'a>(
+            &'a mut self,
+            _roots: GcRootsIter<'a>,
+            _host_data_table: &'a mut ExternRefHostDataTable,
+        ) -> Box<dyn GarbageCollection<'a> + 'a> {
+            return Box::new(NoGc);
+
+            struct NoGc;
+
+            impl<'a> GarbageCollection<'a> for NoGc {
+                fn collect_increment(&mut self) -> GcProgress {
+                    GcProgress::Complete
+                }
+            }
+        }
+        unsafe fn vmctx_gc_heap_base(&self) -> *mut u8 {
+            ptr::null_mut()
+        }
+        unsafe fn vmctx_gc_heap_bound(&self) -> usize {
+            0
+        }
+        unsafe fn vmctx_gc_heap_data(&self) -> *mut u8 {
+            ptr::null_mut()
+        }
+        #[cfg(feature = "pooling-allocator")]
+        fn reset(&mut self) {}
+    }
+}

data/ext/cargo-vendor/wasmtime-21.0.1/src/runtime/vm/imports.rs

@@ -0,0 +1,29 @@
+use crate::runtime::vm::vmcontext::{
+    VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport,
+};
+
+/// Resolved import pointers.
+///
+/// Note that some of these fields are slices, not `PrimaryMap`. They should be
+/// stored in index-order as with the module that we're providing the imports
+/// for, and indexing is all done the same way as the main module's index
+/// spaces.
+///
+/// Also note that the way we compile modules means that for the module linking
+/// proposal all `alias` directives should map to imported items. This means
+/// that each of these items aren't necessarily directly imported, but may be
+/// aliased.
+#[derive(Default)]
+pub struct Imports<'a> {
+    /// Resolved addresses for imported functions.
+    pub functions: &'a [VMFunctionImport],
+
+    /// Resolved addresses for imported tables.
+    pub tables: &'a [VMTableImport],
+
+    /// Resolved addresses for imported memories.
+    pub memories: &'a [VMMemoryImport],
+
+    /// Resolved addresses for imported globals.
+    pub globals: &'a [VMGlobalImport],
+}

data/ext/cargo-vendor/wasmtime-21.0.1/src/runtime/vm/instance/allocator/on_demand.rs

@@ -0,0 +1,218 @@
+use super::{
+    InstanceAllocationRequest, InstanceAllocatorImpl, MemoryAllocationIndex, TableAllocationIndex,
+};
+use crate::prelude::*;
+use crate::runtime::vm::instance::RuntimeMemoryCreator;
+use crate::runtime::vm::memory::{DefaultMemoryCreator, Memory};
+use crate::runtime::vm::mpk::ProtectionKey;
+use crate::runtime::vm::table::Table;
+use crate::runtime::vm::CompiledModuleId;
+use alloc::sync::Arc;
+use anyhow::Result;
+use wasmtime_environ::{
+    DefinedMemoryIndex, DefinedTableIndex, HostPtr, MemoryPlan, Module, TablePlan, VMOffsets,
+};
+
+#[cfg(feature = "gc")]
+use crate::runtime::vm::{GcHeap, GcHeapAllocationIndex, GcRuntime};
+
+#[cfg(feature = "async")]
+use wasmtime_fiber::RuntimeFiberStackCreator;
+
+#[cfg(feature = "component-model")]
+use wasmtime_environ::{
+    component::{Component, VMComponentOffsets},
+    StaticModuleIndex,
+};
+
+/// Represents the on-demand instance allocator.
+#[derive(Clone)]
+pub struct OnDemandInstanceAllocator {
+    mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>,
+    #[cfg(feature = "async")]
+    stack_creator: Option<Arc<dyn RuntimeFiberStackCreator>>,
+    #[cfg(feature = "async")]
+    stack_size: usize,
+}
+
+impl OnDemandInstanceAllocator {
+    /// Creates a new on-demand instance allocator.
+    pub fn new(mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>, stack_size: usize) -> Self {
+        let _ = stack_size; // suppress warnings when async feature is disabled.
+        Self {
+            mem_creator,
+            #[cfg(feature = "async")]
+            stack_creator: None,
+            #[cfg(feature = "async")]
+            stack_size,
+        }
+    }
+
+    /// Set the stack creator.
+    #[cfg(feature = "async")]
+    pub fn set_stack_creator(&mut self, stack_creator: Arc<dyn RuntimeFiberStackCreator>) {
+        self.stack_creator = Some(stack_creator);
+    }
+}
+
+impl Default for OnDemandInstanceAllocator {
+    fn default() -> Self {
+        Self {
+            mem_creator: None,
+            #[cfg(feature = "async")]
+            stack_creator: None,
+            #[cfg(feature = "async")]
+            stack_size: 0,
+        }
+    }
+}
+
+unsafe impl InstanceAllocatorImpl for OnDemandInstanceAllocator {
+    #[cfg(feature = "component-model")]
+    fn validate_component_impl<'a>(
+        &self,
+        _component: &Component,
+        _offsets: &VMComponentOffsets<HostPtr>,
+        _get_module: &'a dyn Fn(StaticModuleIndex) -> &'a Module,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn validate_module_impl(&self, _module: &Module, _offsets: &VMOffsets<HostPtr>) -> Result<()> {
+        Ok(())
+    }
+
+    fn increment_component_instance_count(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn decrement_component_instance_count(&self) {}
+
+    fn increment_core_instance_count(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn decrement_core_instance_count(&self) {}
+
+    unsafe fn allocate_memory(
+        &self,
+        request: &mut InstanceAllocationRequest,
+        memory_plan: &MemoryPlan,
+        memory_index: DefinedMemoryIndex,
+    ) -> Result<(MemoryAllocationIndex, Memory)> {
+        let creator = self
+            .mem_creator
+            .as_deref()
+            .unwrap_or_else(|| &DefaultMemoryCreator);
+        let image = request.runtime_info.memory_image(memory_index)?;
+        let allocation_index = MemoryAllocationIndex::default();
+        let memory = Memory::new_dynamic(
+            memory_plan,
+            creator,
+            request
+                .store
+                .get()
+                .expect("if module has memory plans, store is not empty"),
+            image,
+        )?;
+        Ok((allocation_index, memory))
+    }
+
+    unsafe fn deallocate_memory(
+        &self,
+        _memory_index: DefinedMemoryIndex,
+        allocation_index: MemoryAllocationIndex,
+        _memory: Memory,
+    ) {
+        debug_assert_eq!(allocation_index, MemoryAllocationIndex::default());
+        // Normal destructors do all the necessary clean up.
+    }
+
+    unsafe fn allocate_table(
+        &self,
+        request: &mut InstanceAllocationRequest,
+        table_plan: &TablePlan,
+        _table_index: DefinedTableIndex,
+    ) -> Result<(TableAllocationIndex, Table)> {
+        let allocation_index = TableAllocationIndex::default();
+        let table = Table::new_dynamic(
+            table_plan,
+            request
+                .store
+                .get()
+                .expect("if module has table plans, store is not empty"),
+        )?;
+        Ok((allocation_index, table))
+    }
+
+    unsafe fn deallocate_table(
+        &self,
+        _table_index: DefinedTableIndex,
+        allocation_index: TableAllocationIndex,
+        _table: Table,
+    ) {
+        debug_assert_eq!(allocation_index, TableAllocationIndex::default());
+        // Normal destructors do all the necessary clean up.
+    }
+
+    #[cfg(feature = "async")]
+    fn allocate_fiber_stack(&self) -> Result<wasmtime_fiber::FiberStack> {
+        if self.stack_size == 0 {
+            anyhow::bail!("fiber stacks are not supported by the allocator")
+        }
+        let stack = match &self.stack_creator {
+            Some(stack_creator) => {
+                let stack = stack_creator.new_stack(self.stack_size)?;
+                wasmtime_fiber::FiberStack::from_custom(stack)
+            }
+            None => wasmtime_fiber::FiberStack::new(self.stack_size),
+        }?;
+        Ok(stack)
+    }
+
+    #[cfg(feature = "async")]
+    unsafe fn deallocate_fiber_stack(&self, _stack: &wasmtime_fiber::FiberStack) {
+        // The on-demand allocator has no further bookkeeping for fiber stacks
+    }
+
+    fn purge_module(&self, _: CompiledModuleId) {}
+
+    fn next_available_pkey(&self) -> Option<ProtectionKey> {
+        // The on-demand allocator cannot use protection keys--it requires
+        // back-to-back allocation of memory slots that this allocator cannot
+        // guarantee.
+        None
+    }
+
+    fn restrict_to_pkey(&self, _: ProtectionKey) {
+        // The on-demand allocator cannot use protection keys; an on-demand
+        // allocator will never hand out protection keys to the stores its
+        // engine creates.
+        unreachable!()
+    }
+
+    fn allow_all_pkeys(&self) {
+        // The on-demand allocator cannot use protection keys; an on-demand
+        // allocator will never hand out protection keys to the stores its
+        // engine creates.
+        unreachable!()
+    }
+
+    #[cfg(feature = "gc")]
+    fn allocate_gc_heap(
+        &self,
+        gc_runtime: &dyn GcRuntime,
+    ) -> Result<(GcHeapAllocationIndex, Box<dyn GcHeap>)> {
+        Ok((GcHeapAllocationIndex::default(), gc_runtime.new_gc_heap()?))
+    }
+
+    #[cfg(feature = "gc")]
+    fn deallocate_gc_heap(
+        &self,
+        allocation_index: GcHeapAllocationIndex,
+        gc_heap: Box<dyn crate::runtime::vm::GcHeap>,
+    ) {
+        debug_assert_eq!(allocation_index, GcHeapAllocationIndex::default());
+        drop(gc_heap);
+    }
+}

data/ext/cargo-vendor/wasmtime-21.0.1/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs

@@ -0,0 +1,93 @@
+use super::index_allocator::{SimpleIndexAllocator, SlotId};
+use super::GcHeapAllocationIndex;
+use crate::prelude::*;
+use crate::runtime::vm::{GcHeap, GcRuntime, PoolingInstanceAllocatorConfig, Result};
+use anyhow::anyhow;
+use std::sync::Mutex;
+
+/// A pool of reusable GC heaps.
+pub struct GcHeapPool {
+    max_gc_heaps: usize,
+    index_allocator: SimpleIndexAllocator,
+    heaps: Mutex<Vec<Option<Box<dyn GcHeap>>>>,
+}
+
+impl std::fmt::Debug for GcHeapPool {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GcHeapPool")
+            .field("max_gc_heaps", &self.max_gc_heaps)
+            .field("index_allocator", &self.index_allocator)
+            .field("heaps", &"..")
+            .finish()
+    }
+}
+
+impl GcHeapPool {
+    /// Create a new `GcHeapPool` with the given configuration.
+    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
+        let index_allocator = SimpleIndexAllocator::new(config.limits.total_gc_heaps);
+        let max_gc_heaps = usize::try_from(config.limits.total_gc_heaps).unwrap();
+
+        // Each individual GC heap in the pool is lazily allocated. See the
+        // `allocate` method.
+        let heaps = Mutex::new((0..max_gc_heaps).map(|_| None).collect());
+
+        Ok(Self {
+            max_gc_heaps,
+            index_allocator,
+            heaps,
+        })
+    }
+
+    /// Are there zero slots in use right now?
+    pub fn is_empty(&self) -> bool {
+        self.index_allocator.is_empty()
+    }
+
+    /// Allocate a single table for the given instance allocation request.
+    pub fn allocate(
+        &self,
+        gc_runtime: &dyn GcRuntime,
+    ) -> Result<(GcHeapAllocationIndex, Box<dyn GcHeap>)> {
+        let allocation_index = self
+            .index_allocator
+            .alloc()
+            .map(|slot| GcHeapAllocationIndex(slot.0))
+            .ok_or_else(|| {
+                anyhow!(
+                    "maximum concurrent GC heap limit of {} reached",
+                    self.max_gc_heaps
+                )
+            })?;
+
+        let heap = match {
+            let mut heaps = self.heaps.lock().unwrap();
+            heaps[allocation_index.index()].take()
+        } {
+            // If we already have a heap at this slot, reuse it.
+            Some(heap) => heap,
+            // Otherwise, we haven't forced this slot's lazily allocated heap
+            // yet. So do that now.
+            None => gc_runtime.new_gc_heap()?,
+        };
+
+        Ok((allocation_index, heap))
+    }
+
+    /// Deallocate a previously-allocated GC heap.
+    pub fn deallocate(&self, allocation_index: GcHeapAllocationIndex, mut heap: Box<dyn GcHeap>) {
+        heap.reset();
+
+        // NB: Replace the heap before freeing the index. If we did it in the
+        // opposite order, a concurrent allocation request could reallocate the
+        // index before we have replaced the heap.
+
+        {
+            let mut heaps = self.heaps.lock().unwrap();
+            let old_entry = std::mem::replace(&mut heaps[allocation_index.index()], Some(heap));
+            debug_assert!(old_entry.is_none());
+        }
+
+        self.index_allocator.free(SlotId(allocation_index.0));
+    }
+}

data/ext/cargo-vendor/wasmtime-21.0.1/src/runtime/vm/instance/allocator/pooling/generic_stack_pool.rs

@@ -0,0 +1,66 @@
+#![cfg_attr(not(asan), allow(dead_code))]
+
+use crate::runtime::vm::PoolingInstanceAllocatorConfig;
+use anyhow::{bail, Result};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+/// A generic implementation of a stack pool.
+///
+/// This implementation technically doesn't actually pool anything at this time.
+/// Originally this was the implementation for non-Unix (e.g. Windows and
+/// MIRI), but nowadays this is also used for fuzzing. For more documentation
+/// for why this is used on fuzzing see the `asan` module in the
+/// `wasmtime-fiber` crate.
+///
+/// Currently the only purpose of `StackPool` is to limit the total number of
+/// concurrent stacks while otherwise leveraging `wasmtime_fiber::FiberStack`
+/// natively.
+#[derive(Debug)]
+pub struct StackPool {
+    stack_size: usize,
+    live_stacks: AtomicU64,
+    stack_limit: u64,
+}
+
+impl StackPool {
+    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
+        Ok(StackPool {
+            stack_size: config.stack_size,
+            live_stacks: AtomicU64::new(0),
+            stack_limit: config.limits.total_stacks.into(),
+        })
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.live_stacks.load(Ordering::Acquire) == 0
+    }
+
+    pub fn allocate(&self) -> Result<wasmtime_fiber::FiberStack> {
+        if self.stack_size == 0 {
+            bail!("fiber stack allocation not supported")
+        }
+
+        let old_count = self.live_stacks.fetch_add(1, Ordering::AcqRel);
+        if old_count >= self.stack_limit {
+            self.live_stacks.fetch_sub(1, Ordering::AcqRel);
+            bail!(
+                "maximum concurrent fiber limit of {} reached",
+                self.stack_limit
+            );
+        }
+
+        match wasmtime_fiber::FiberStack::new(self.stack_size) {
+            Ok(stack) => Ok(stack),
+            Err(e) => {
+                self.live_stacks.fetch_sub(1, Ordering::AcqRel);
+                Err(anyhow::Error::from(e))
+            }
+        }
+    }
+
+    pub unsafe fn deallocate(&self, stack: &wasmtime_fiber::FiberStack) {
+        self.live_stacks.fetch_sub(1, Ordering::AcqRel);
+        // A no-op as we don't own the fiber stack on Windows.
+        let _ = stack;
+    }
+}