wasmtime 15.0.1 → 16.0.0

data/ext/cargo-vendor/wasmtime-runtime-15.0.1/src/instance/allocator/pooling/stack_pool.rs

@@ -1,242 +0,0 @@
-use super::{
-    imp::{commit_stack_pages, reset_stack_pages_to_zero},
-    index_allocator::{SimpleIndexAllocator, SlotId},
-    round_up_to_pow2,
-};
-use crate::{Mmap, PoolingInstanceAllocatorConfig};
-use anyhow::{anyhow, bail, Context, Result};
-
-/// Represents a pool of execution stacks (used for the async fiber implementation).
-///
-/// Each index into the pool represents a single execution stack. The maximum number of
-/// stacks is the same as the maximum number of instances.
-///
-/// As stacks grow downwards, each stack starts (lowest address) with a guard page
-/// that can be used to detect stack overflow.
-///
-/// The top of the stack (starting stack pointer) is returned when a stack is allocated
-/// from the pool.
-#[derive(Debug)]
-pub struct StackPool {
-    mapping: Mmap,
-    stack_size: usize,
-    max_stacks: usize,
-    page_size: usize,
-    index_allocator: SimpleIndexAllocator,
-    async_stack_zeroing: bool,
-    async_stack_keep_resident: usize,
-}
-
-impl StackPool {
-    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
-        use rustix::mm::{mprotect, MprotectFlags};
-
-        let page_size = crate::page_size();
-
-        // Add a page to the stack size for the guard page when using fiber stacks
-        let stack_size = if config.stack_size == 0 {
-            0
-        } else {
-            round_up_to_pow2(config.stack_size, page_size)
-                .checked_add(page_size)
-                .ok_or_else(|| anyhow!("stack size exceeds addressable memory"))?
-        };
-
-        let max_stacks = usize::try_from(config.limits.total_stacks).unwrap();
-
-        let allocation_size = stack_size
-            .checked_mul(max_stacks)
-            .ok_or_else(|| anyhow!("total size of execution stacks exceeds addressable memory"))?;
-
-        let mapping = Mmap::accessible_reserved(allocation_size, allocation_size)
-            .context("failed to create stack pool mapping")?;
-
-        // Set up the stack guard pages.
-        if allocation_size > 0 {
-            unsafe {
-                for i in 0..max_stacks {
-                    // Make the stack guard page inaccessible.
-                    let bottom_of_stack = mapping.as_ptr().add(i * stack_size).cast_mut();
-                    mprotect(bottom_of_stack.cast(), page_size, MprotectFlags::empty())
-                        .context("failed to protect stack guard page")?;
-                }
-            }
-        }
-
-        Ok(Self {
-            mapping,
-            stack_size,
-            max_stacks,
-            page_size,
-            async_stack_zeroing: config.async_stack_zeroing,
-            async_stack_keep_resident: config.async_stack_keep_resident,
-            index_allocator: SimpleIndexAllocator::new(config.limits.total_stacks),
-        })
-    }
-
-    /// Are there zero slots in use right now?
-    pub fn is_empty(&self) -> bool {
-        self.index_allocator.is_empty()
-    }
-
-    /// Allocate a new fiber.
-    pub fn allocate(&self) -> Result<wasmtime_fiber::FiberStack> {
-        if self.stack_size == 0 {
-            bail!("pooling allocator not configured to enable fiber stack allocation");
-        }
-
-        let index = self
-            .index_allocator
-            .alloc()
-            .ok_or_else(|| {
-                anyhow!(
-                    "maximum concurrent fiber limit of {} reached",
-                    self.max_stacks
-                )
-            })?
-            .index();
-
-        assert!(index < self.max_stacks);
-
-        unsafe {
-            // Remove the guard page from the size
-            let size_without_guard = self.stack_size - self.page_size;
-
-            let bottom_of_stack = self
-                .mapping
-                .as_ptr()
-                .add((index * self.stack_size) + self.page_size)
-                .cast_mut();
-
-            commit_stack_pages(bottom_of_stack, size_without_guard)?;
-
-            let stack =
-                wasmtime_fiber::FiberStack::from_raw_parts(bottom_of_stack, size_without_guard)?;
-            Ok(stack)
-        }
-    }
-
-    /// Deallocate a previously-allocated fiber.
-    ///
-    /// # Safety
-    ///
-    /// The fiber must have been allocated by this pool, must be in an allocated
-    /// state, and must never be used again.
-    pub unsafe fn deallocate(&self, stack: &wasmtime_fiber::FiberStack) {
-        let top = stack
-            .top()
-            .expect("fiber stack not allocated from the pool") as usize;
-
-        let base = self.mapping.as_ptr() as usize;
-        let len = self.mapping.len();
-        assert!(
-            top > base && top <= (base + len),
-            "fiber stack top pointer not in range"
-        );
-
-        // Remove the guard page from the size
-        let stack_size = self.stack_size - self.page_size;
-        let bottom_of_stack = top - stack_size;
-        let start_of_stack = bottom_of_stack - self.page_size;
-        assert!(start_of_stack >= base && start_of_stack < (base + len));
-        assert!((start_of_stack - base) % self.stack_size == 0);
-
-        let index = (start_of_stack - base) / self.stack_size;
-        assert!(index < self.max_stacks);
-
-        if self.async_stack_zeroing {
-            self.zero_stack(bottom_of_stack, stack_size);
-        }
-
-        self.index_allocator.free(SlotId(index as u32));
-    }
-
-    fn zero_stack(&self, bottom: usize, size: usize) {
-        // Manually zero the top of the stack to keep the pages resident in
-        // memory and avoid future page faults. Use the system to deallocate
-        // pages past this. This hopefully strikes a reasonable balance between:
-        //
-        // * memset for the whole range is probably expensive
-        // * madvise for the whole range incurs expensive future page faults
-        // * most threads probably don't use most of the stack anyway
-        let size_to_memset = size.min(self.async_stack_keep_resident);
-        unsafe {
-            std::ptr::write_bytes(
-                (bottom + size - size_to_memset) as *mut u8,
-                0,
-                size_to_memset,
-            );
-        }
-
-        // Use the system to reset remaining stack pages to zero.
-        reset_stack_pages_to_zero(bottom as _, size - size_to_memset).unwrap();
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::InstanceLimits;
-
-    #[cfg(all(unix, target_pointer_width = "64", feature = "async", not(miri)))]
-    #[test]
-    fn test_stack_pool() -> Result<()> {
-        let config = PoolingInstanceAllocatorConfig {
-            limits: InstanceLimits {
-                total_stacks: 10,
-                ..Default::default()
-            },
-            stack_size: 1,
-            async_stack_zeroing: true,
-            ..PoolingInstanceAllocatorConfig::default()
-        };
-        let pool = StackPool::new(&config)?;
-
-        let native_page_size = crate::page_size();
-        assert_eq!(pool.stack_size, 2 * native_page_size);
-        assert_eq!(pool.max_stacks, 10);
-        assert_eq!(pool.page_size, native_page_size);
-
-        assert_eq!(pool.index_allocator.testing_freelist(), []);
-
-        let base = pool.mapping.as_ptr() as usize;
-
-        let mut stacks = Vec::new();
-        for i in 0..10 {
-            let stack = pool.allocate().expect("allocation should succeed");
-            assert_eq!(
-                ((stack.top().unwrap() as usize - base) / pool.stack_size) - 1,
-                i
-            );
-            stacks.push(stack);
-        }
-
-        assert_eq!(pool.index_allocator.testing_freelist(), []);
-
-        assert!(pool.allocate().is_err(), "allocation should fail");
-
-        for stack in stacks {
-            unsafe {
-                pool.deallocate(&stack);
-            }
-        }
-
-        assert_eq!(
-            pool.index_allocator.testing_freelist(),
-            [
-                SlotId(0),
-                SlotId(1),
-                SlotId(2),
-                SlotId(3),
-                SlotId(4),
-                SlotId(5),
-                SlotId(6),
-                SlotId(7),
-                SlotId(8),
-                SlotId(9)
-            ],
-        );
-
-        Ok(())
-    }
-}

data/ext/cargo-vendor/wasmtime-runtime-15.0.1/src/instance/allocator/pooling/table_pool.rs

@@ -1,225 +0,0 @@
-use super::{
-    imp::{commit_table_pages, decommit_table_pages},
-    index_allocator::{SimpleIndexAllocator, SlotId},
-    round_up_to_pow2, TableAllocationIndex,
-};
-use crate::{InstanceAllocationRequest, Mmap, PoolingInstanceAllocatorConfig, SendSyncPtr, Table};
-use anyhow::{anyhow, bail, Context, Result};
-use std::mem;
-use std::ptr::NonNull;
-use wasmtime_environ::{Module, TablePlan};
-
-/// Represents a pool of WebAssembly tables.
-///
-/// Each instance index into the pool returns an iterator over the base addresses
-/// of the instance's tables.
-#[derive(Debug)]
-pub struct TablePool {
-    index_allocator: SimpleIndexAllocator,
-    mapping: Mmap,
-    table_size: usize,
-    max_total_tables: usize,
-    tables_per_instance: usize,
-    page_size: usize,
-    keep_resident: usize,
-    table_elements: usize,
-}
-
-impl TablePool {
-    /// Create a new `TablePool`.
-    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
-        let page_size = crate::page_size();
-
-        let table_size = round_up_to_pow2(
-            mem::size_of::<*mut u8>()
-                .checked_mul(config.limits.table_elements as usize)
-                .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?,
-            page_size,
-        );
-
-        let max_total_tables = usize::try_from(config.limits.total_tables).unwrap();
-        let tables_per_instance = usize::try_from(config.limits.max_tables_per_module).unwrap();
-
-        let allocation_size = table_size
-            .checked_mul(max_total_tables)
-            .ok_or_else(|| anyhow!("total size of tables exceeds addressable memory"))?;
-
-        let mapping = Mmap::accessible_reserved(allocation_size, allocation_size)
-            .context("failed to create table pool mapping")?;
-
-        Ok(Self {
-            index_allocator: SimpleIndexAllocator::new(config.limits.total_tables),
-            mapping,
-            table_size,
-            max_total_tables,
-            tables_per_instance,
-            page_size,
-            keep_resident: config.table_keep_resident,
-            table_elements: usize::try_from(config.limits.table_elements).unwrap(),
-        })
-    }
-
-    /// Validate whether this module's tables are allocatable by this pool.
-    pub fn validate(&self, module: &Module) -> Result<()> {
-        let tables = module.table_plans.len() - module.num_imported_tables;
-
-        if tables > usize::try_from(self.tables_per_instance).unwrap() {
-            bail!(
-                "defined tables count of {} exceeds the per-instance limit of {}",
-                tables,
-                self.tables_per_instance,
-            );
-        }
-
-        if tables > self.max_total_tables {
-            bail!(
-                "defined tables count of {} exceeds the total tables limit of {}",
-                tables,
-                self.max_total_tables,
-            );
-        }
-
-        for (i, plan) in module.table_plans.iter().skip(module.num_imported_tables) {
-            if plan.table.minimum > u32::try_from(self.table_elements).unwrap() {
-                bail!(
-                    "table index {} has a minimum element size of {} which exceeds the limit of {}",
-                    i.as_u32(),
-                    plan.table.minimum,
-                    self.table_elements,
-                );
-            }
-        }
-        Ok(())
-    }
-
-    /// Are there zero slots in use right now?
-    pub fn is_empty(&self) -> bool {
-        self.index_allocator.is_empty()
-    }
-
-    /// Get the base pointer of the given table allocation.
-    fn get(&self, table_index: TableAllocationIndex) -> *mut u8 {
-        assert!(table_index.index() < self.max_total_tables);
-
-        unsafe {
-            self.mapping
-                .as_ptr()
-                .add(table_index.index() * self.table_size)
-                .cast_mut()
-        }
-    }
-
-    /// Allocate a single table for the given instance allocation request.
-    pub fn allocate(
-        &self,
-        request: &mut InstanceAllocationRequest,
-        table_plan: &TablePlan,
-    ) -> Result<(TableAllocationIndex, Table)> {
-        let allocation_index = self
-            .index_allocator
-            .alloc()
-            .map(|slot| TableAllocationIndex(slot.0))
-            .ok_or_else(|| {
-                anyhow!(
-                    "maximum concurrent table limit of {} reached",
-                    self.max_total_tables
-                )
-            })?;
-
-        match (|| {
-            let base = self.get(allocation_index);
-
-            commit_table_pages(
-                base as *mut u8,
-                self.table_elements * mem::size_of::<*mut u8>(),
-            )?;
-
-            let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut(
-                base.cast(),
-                self.table_elements,
-            ))
-            .unwrap();
-            Table::new_static(table_plan, SendSyncPtr::new(ptr), unsafe {
-                &mut *request.store.get().unwrap()
-            })
-        })() {
-            Ok(table) => Ok((allocation_index, table)),
-            Err(e) => {
-                self.index_allocator.free(SlotId(allocation_index.0));
-                Err(e)
-            }
-        }
-    }
-
-    /// Deallocate a previously-allocated table.
-    ///
-    /// # Safety
-    ///
-    /// The table must have been previously-allocated by this pool and assigned
-    /// the given allocation index, it must currently be allocated, and it must
-    /// never be used again.
-    pub unsafe fn deallocate(&self, allocation_index: TableAllocationIndex, table: Table) {
-        assert!(table.is_static());
-
-        let size = round_up_to_pow2(
-            table.size() as usize * mem::size_of::<*mut u8>(),
-            self.page_size,
-        );
-
-        drop(table);
-
-        let base = self.get(allocation_index);
-        self.reset_table_pages_to_zero(base, size)
-            .expect("failed to decommit table pages");
-
-        self.index_allocator.free(SlotId(allocation_index.0));
-    }
-
-    fn reset_table_pages_to_zero(&self, base: *mut u8, size: usize) -> Result<()> {
-        let size_to_memset = size.min(self.keep_resident);
-        unsafe {
-            std::ptr::write_bytes(base, 0, size_to_memset);
-            decommit_table_pages(base.add(size_to_memset), size - size_to_memset)
-                .context("failed to decommit table page")?;
-        }
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::InstanceLimits;
-
-    #[cfg(target_pointer_width = "64")]
-    #[test]
-    fn test_table_pool() -> Result<()> {
-        let pool = TablePool::new(&PoolingInstanceAllocatorConfig {
-            limits: InstanceLimits {
-                total_tables: 7,
-                table_elements: 100,
-                memory_pages: 0,
-                max_memories_per_module: 0,
-                ..Default::default()
-            },
-            ..Default::default()
-        })?;
-
-        let host_page_size = crate::page_size();
-
-        assert_eq!(pool.table_size, host_page_size);
-        assert_eq!(pool.max_total_tables, 7);
-        assert_eq!(pool.page_size, host_page_size);
-        assert_eq!(pool.table_elements, 100);
-
-        let base = pool.mapping.as_ptr() as usize;
-
-        for i in 0..7 {
-            let index = TableAllocationIndex(i);
-            let ptr = pool.get(index);
-            assert_eq!(ptr as usize - base, i as usize * pool.table_size);
-        }
-
-        Ok(())
-    }
-}

data/ext/cargo-vendor/wasmtime-runtime-15.0.1/src/instance/allocator/pooling/unix.rs

@@ -1,56 +0,0 @@
-use anyhow::Result;
-
-fn decommit(addr: *mut u8, len: usize) -> Result<()> {
-    if len == 0 {
-        return Ok(());
-    }
-
-    unsafe {
-        cfg_if::cfg_if! {
-            if #[cfg(miri)] {
-                std::ptr::write_bytes(addr, 0, len);
-            } else if #[cfg(target_os = "linux")] {
-                use rustix::mm::{madvise, Advice};
-
-                // On Linux, this is enough to cause the kernel to initialize
-                // the pages to 0 on next access
-                madvise(addr as _, len, Advice::LinuxDontNeed)?;
-            } else {
-                use rustix::mm::{mmap_anonymous, ProtFlags, MapFlags};
-
-                // By creating a new mapping at the same location, this will
-                // discard the mapping for the pages in the given range.
-                // The new mapping will be to the CoW zero page, so this
-                // effectively zeroes the pages.
-                mmap_anonymous(
-                    addr as _,
-                    len,
-                    ProtFlags::READ | ProtFlags::WRITE,
-                    MapFlags::PRIVATE | MapFlags::FIXED,
-                )?;
-            }
-        }
-    }
-
-    Ok(())
-}
-
-pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> {
-    // A no-op as table pages remain READ|WRITE
-    Ok(())
-}
-
-pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> {
-    decommit(addr, len)
-}
-
-#[cfg(all(feature = "async", not(miri)))]
-pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> {
-    // A no-op as stack pages remain READ|WRITE
-    Ok(())
-}
-
-#[cfg(all(feature = "async", not(miri)))]
-pub fn reset_stack_pages_to_zero(addr: *mut u8, len: usize) -> Result<()> {
-    decommit(addr, len)
-}

data/ext/cargo-vendor/wasmtime-runtime-15.0.1/src/instance/allocator/pooling/windows.rs

@@ -1,38 +0,0 @@
-use anyhow::{bail, Result};
-use windows_sys::Win32::System::Memory::*;
-
-pub fn commit(addr: *mut u8, len: usize) -> Result<()> {
-    if len == 0 {
-        return Ok(());
-    }
-
-    // Memory needs to be committed, so don't use the `region` crate
-    if unsafe { VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() } {
-        bail!("failed to commit memory as read/write");
-    }
-
-    Ok(())
-}
-
-pub fn decommit(addr: *mut u8, len: usize) -> Result<()> {
-    if len == 0 {
-        return Ok(());
-    }
-
-    if unsafe { VirtualFree(addr as _, len, MEM_DECOMMIT) } == 0 {
-        bail!(
-            "failed to decommit memory pages: {}",
-            std::io::Error::last_os_error()
-        );
-    }
-
-    Ok(())
-}
-
-pub fn commit_table_pages(addr: *mut u8, len: usize) -> Result<()> {
-    commit(addr, len)
-}
-
-pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> {
-    decommit(addr, len)
-}