wasmtime 15.0.1 → 16.0.0

data/ext/cargo-vendor/wasmtime-runtime-16.0.0/src/instance/allocator/pooling/stack_pool.rs

@@ -0,0 +1,242 @@
+use super::{
+    index_allocator::{SimpleIndexAllocator, SlotId},
+    round_up_to_pow2,
+};
+use crate::sys::vm::{commit_stack_pages, reset_stack_pages_to_zero};
+use crate::{Mmap, PoolingInstanceAllocatorConfig};
+use anyhow::{anyhow, bail, Context, Result};
+
+/// Represents a pool of execution stacks (used for the async fiber implementation).
+///
+/// Each index into the pool represents a single execution stack. The maximum number of
+/// stacks is the same as the maximum number of instances.
+///
+/// As stacks grow downwards, each stack starts (lowest address) with a guard page
+/// that can be used to detect stack overflow.
+///
+/// The top of the stack (starting stack pointer) is returned when a stack is allocated
+/// from the pool.
+#[derive(Debug)]
+pub struct StackPool {
+    mapping: Mmap,
+    stack_size: usize,
+    max_stacks: usize,
+    page_size: usize,
+    index_allocator: SimpleIndexAllocator,
+    async_stack_zeroing: bool,
+    async_stack_keep_resident: usize,
+}
+
+impl StackPool {
+    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
+        use rustix::mm::{mprotect, MprotectFlags};
+
+        let page_size = crate::page_size();
+
+        // Add a page to the stack size for the guard page when using fiber stacks
+        let stack_size = if config.stack_size == 0 {
+            0
+        } else {
+            round_up_to_pow2(config.stack_size, page_size)
+                .checked_add(page_size)
+                .ok_or_else(|| anyhow!("stack size exceeds addressable memory"))?
+        };
+
+        let max_stacks = usize::try_from(config.limits.total_stacks).unwrap();
+
+        let allocation_size = stack_size
+            .checked_mul(max_stacks)
+            .ok_or_else(|| anyhow!("total size of execution stacks exceeds addressable memory"))?;
+
+        let mapping = Mmap::accessible_reserved(allocation_size, allocation_size)
+            .context("failed to create stack pool mapping")?;
+
+        // Set up the stack guard pages.
+        if allocation_size > 0 {
+            unsafe {
+                for i in 0..max_stacks {
+                    // Make the stack guard page inaccessible.
+                    let bottom_of_stack = mapping.as_ptr().add(i * stack_size).cast_mut();
+                    mprotect(bottom_of_stack.cast(), page_size, MprotectFlags::empty())
+                        .context("failed to protect stack guard page")?;
+                }
+            }
+        }
+
+        Ok(Self {
+            mapping,
+            stack_size,
+            max_stacks,
+            page_size,
+            async_stack_zeroing: config.async_stack_zeroing,
+            async_stack_keep_resident: config.async_stack_keep_resident,
+            index_allocator: SimpleIndexAllocator::new(config.limits.total_stacks),
+        })
+    }
+
+    /// Are there zero slots in use right now?
+    pub fn is_empty(&self) -> bool {
+        self.index_allocator.is_empty()
+    }
+
+    /// Allocate a new fiber.
+    pub fn allocate(&self) -> Result<wasmtime_fiber::FiberStack> {
+        if self.stack_size == 0 {
+            bail!("pooling allocator not configured to enable fiber stack allocation");
+        }
+
+        let index = self
+            .index_allocator
+            .alloc()
+            .ok_or_else(|| {
+                anyhow!(
+                    "maximum concurrent fiber limit of {} reached",
+                    self.max_stacks
+                )
+            })?
+            .index();
+
+        assert!(index < self.max_stacks);
+
+        unsafe {
+            // Remove the guard page from the size
+            let size_without_guard = self.stack_size - self.page_size;
+
+            let bottom_of_stack = self
+                .mapping
+                .as_ptr()
+                .add((index * self.stack_size) + self.page_size)
+                .cast_mut();
+
+            commit_stack_pages(bottom_of_stack, size_without_guard)?;
+
+            let stack =
+                wasmtime_fiber::FiberStack::from_raw_parts(bottom_of_stack, size_without_guard)?;
+            Ok(stack)
+        }
+    }
+
+    /// Deallocate a previously-allocated fiber.
+    ///
+    /// # Safety
+    ///
+    /// The fiber must have been allocated by this pool, must be in an allocated
+    /// state, and must never be used again.
+    pub unsafe fn deallocate(&self, stack: &wasmtime_fiber::FiberStack) {
+        let top = stack
+            .top()
+            .expect("fiber stack not allocated from the pool") as usize;
+
+        let base = self.mapping.as_ptr() as usize;
+        let len = self.mapping.len();
+        assert!(
+            top > base && top <= (base + len),
+            "fiber stack top pointer not in range"
+        );
+
+        // Remove the guard page from the size
+        let stack_size = self.stack_size - self.page_size;
+        let bottom_of_stack = top - stack_size;
+        let start_of_stack = bottom_of_stack - self.page_size;
+        assert!(start_of_stack >= base && start_of_stack < (base + len));
+        assert!((start_of_stack - base) % self.stack_size == 0);
+
+        let index = (start_of_stack - base) / self.stack_size;
+        assert!(index < self.max_stacks);
+
+        if self.async_stack_zeroing {
+            self.zero_stack(bottom_of_stack, stack_size);
+        }
+
+        self.index_allocator.free(SlotId(index as u32));
+    }
+
+    fn zero_stack(&self, bottom: usize, size: usize) {
+        // Manually zero the top of the stack to keep the pages resident in
+        // memory and avoid future page faults. Use the system to deallocate
+        // pages past this. This hopefully strikes a reasonable balance between:
+        //
+        // * memset for the whole range is probably expensive
+        // * madvise for the whole range incurs expensive future page faults
+        // * most threads probably don't use most of the stack anyway
+        let size_to_memset = size.min(self.async_stack_keep_resident);
+        unsafe {
+            std::ptr::write_bytes(
+                (bottom + size - size_to_memset) as *mut u8,
+                0,
+                size_to_memset,
+            );
+
+            // Use the system to reset remaining stack pages to zero.
+            reset_stack_pages_to_zero(bottom as _, size - size_to_memset).unwrap();
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::InstanceLimits;
+
+    #[cfg(all(unix, target_pointer_width = "64", feature = "async", not(miri)))]
+    #[test]
+    fn test_stack_pool() -> Result<()> {
+        let config = PoolingInstanceAllocatorConfig {
+            limits: InstanceLimits {
+                total_stacks: 10,
+                ..Default::default()
+            },
+            stack_size: 1,
+            async_stack_zeroing: true,
+            ..PoolingInstanceAllocatorConfig::default()
+        };
+        let pool = StackPool::new(&config)?;
+
+        let native_page_size = crate::page_size();
+        assert_eq!(pool.stack_size, 2 * native_page_size);
+        assert_eq!(pool.max_stacks, 10);
+        assert_eq!(pool.page_size, native_page_size);
+
+        assert_eq!(pool.index_allocator.testing_freelist(), []);
+
+        let base = pool.mapping.as_ptr() as usize;
+
+        let mut stacks = Vec::new();
+        for i in 0..10 {
+            let stack = pool.allocate().expect("allocation should succeed");
+            assert_eq!(
+                ((stack.top().unwrap() as usize - base) / pool.stack_size) - 1,
+                i
+            );
+            stacks.push(stack);
+        }
+
+        assert_eq!(pool.index_allocator.testing_freelist(), []);
+
+        assert!(pool.allocate().is_err(), "allocation should fail");
+
+        for stack in stacks {
+            unsafe {
+                pool.deallocate(&stack);
+            }
+        }
+
+        assert_eq!(
+            pool.index_allocator.testing_freelist(),
+            [
+                SlotId(0),
+                SlotId(1),
+                SlotId(2),
+                SlotId(3),
+                SlotId(4),
+                SlotId(5),
+                SlotId(6),
+                SlotId(7),
+                SlotId(8),
+                SlotId(9)
+            ],
+        );
+
+        Ok(())
+    }
+}

data/ext/cargo-vendor/wasmtime-runtime-16.0.0/src/instance/allocator/pooling/table_pool.rs

@@ -0,0 +1,227 @@
+use super::{
+    index_allocator::{SimpleIndexAllocator, SlotId},
+    round_up_to_pow2, TableAllocationIndex,
+};
+use crate::sys::vm::{commit_table_pages, decommit_table_pages};
+use crate::{InstanceAllocationRequest, Mmap, PoolingInstanceAllocatorConfig, SendSyncPtr, Table};
+use anyhow::{anyhow, bail, Context, Result};
+use std::mem;
+use std::ptr::NonNull;
+use wasmtime_environ::{Module, TablePlan};
+
+/// Represents a pool of WebAssembly tables.
+///
+/// Each instance index into the pool returns an iterator over the base addresses
+/// of the instance's tables.
+#[derive(Debug)]
+pub struct TablePool {
+    index_allocator: SimpleIndexAllocator,
+    mapping: Mmap,
+    table_size: usize,
+    max_total_tables: usize,
+    tables_per_instance: usize,
+    page_size: usize,
+    keep_resident: usize,
+    table_elements: usize,
+}
+
+impl TablePool {
+    /// Create a new `TablePool`.
+    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
+        let page_size = crate::page_size();
+
+        let table_size = round_up_to_pow2(
+            mem::size_of::<*mut u8>()
+                .checked_mul(config.limits.table_elements as usize)
+                .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?,
+            page_size,
+        );
+
+        let max_total_tables = usize::try_from(config.limits.total_tables).unwrap();
+        let tables_per_instance = usize::try_from(config.limits.max_tables_per_module).unwrap();
+
+        let allocation_size = table_size
+            .checked_mul(max_total_tables)
+            .ok_or_else(|| anyhow!("total size of tables exceeds addressable memory"))?;
+
+        let mapping = Mmap::accessible_reserved(allocation_size, allocation_size)
+            .context("failed to create table pool mapping")?;
+
+        Ok(Self {
+            index_allocator: SimpleIndexAllocator::new(config.limits.total_tables),
+            mapping,
+            table_size,
+            max_total_tables,
+            tables_per_instance,
+            page_size,
+            keep_resident: config.table_keep_resident,
+            table_elements: usize::try_from(config.limits.table_elements).unwrap(),
+        })
+    }
+
+    /// Validate whether this module's tables are allocatable by this pool.
+    pub fn validate(&self, module: &Module) -> Result<()> {
+        let tables = module.table_plans.len() - module.num_imported_tables;
+
+        if tables > usize::try_from(self.tables_per_instance).unwrap() {
+            bail!(
+                "defined tables count of {} exceeds the per-instance limit of {}",
+                tables,
+                self.tables_per_instance,
+            );
+        }
+
+        if tables > self.max_total_tables {
+            bail!(
+                "defined tables count of {} exceeds the total tables limit of {}",
+                tables,
+                self.max_total_tables,
+            );
+        }
+
+        for (i, plan) in module.table_plans.iter().skip(module.num_imported_tables) {
+            if plan.table.minimum > u32::try_from(self.table_elements).unwrap() {
+                bail!(
+                    "table index {} has a minimum element size of {} which exceeds the limit of {}",
+                    i.as_u32(),
+                    plan.table.minimum,
+                    self.table_elements,
+                );
+            }
+        }
+        Ok(())
+    }
+
+    /// Are there zero slots in use right now?
+    pub fn is_empty(&self) -> bool {
+        self.index_allocator.is_empty()
+    }
+
+    /// Get the base pointer of the given table allocation.
+    fn get(&self, table_index: TableAllocationIndex) -> *mut u8 {
+        assert!(table_index.index() < self.max_total_tables);
+
+        unsafe {
+            self.mapping
+                .as_ptr()
+                .add(table_index.index() * self.table_size)
+                .cast_mut()
+        }
+    }
+
+    /// Allocate a single table for the given instance allocation request.
+    pub fn allocate(
+        &self,
+        request: &mut InstanceAllocationRequest,
+        table_plan: &TablePlan,
+    ) -> Result<(TableAllocationIndex, Table)> {
+        let allocation_index = self
+            .index_allocator
+            .alloc()
+            .map(|slot| TableAllocationIndex(slot.0))
+            .ok_or_else(|| {
+                anyhow!(
+                    "maximum concurrent table limit of {} reached",
+                    self.max_total_tables
+                )
+            })?;
+
+        match (|| {
+            let base = self.get(allocation_index);
+
+            unsafe {
+                commit_table_pages(
+                    base as *mut u8,
+                    self.table_elements * mem::size_of::<*mut u8>(),
+                )?;
+            }
+
+            let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut(
+                base.cast(),
+                self.table_elements,
+            ))
+            .unwrap();
+            Table::new_static(table_plan, SendSyncPtr::new(ptr), unsafe {
+                &mut *request.store.get().unwrap()
+            })
+        })() {
+            Ok(table) => Ok((allocation_index, table)),
+            Err(e) => {
+                self.index_allocator.free(SlotId(allocation_index.0));
+                Err(e)
+            }
+        }
+    }
+
+    /// Deallocate a previously-allocated table.
+    ///
+    /// # Safety
+    ///
+    /// The table must have been previously-allocated by this pool and assigned
+    /// the given allocation index, it must currently be allocated, and it must
+    /// never be used again.
+    pub unsafe fn deallocate(&self, allocation_index: TableAllocationIndex, table: Table) {
+        assert!(table.is_static());
+
+        let size = round_up_to_pow2(
+            table.size() as usize * mem::size_of::<*mut u8>(),
+            self.page_size,
+        );
+
+        drop(table);
+
+        let base = self.get(allocation_index);
+        self.reset_table_pages_to_zero(base, size)
+            .expect("failed to decommit table pages");
+
+        self.index_allocator.free(SlotId(allocation_index.0));
+    }
+
+    fn reset_table_pages_to_zero(&self, base: *mut u8, size: usize) -> Result<()> {
+        let size_to_memset = size.min(self.keep_resident);
+        unsafe {
+            std::ptr::write_bytes(base, 0, size_to_memset);
+            decommit_table_pages(base.add(size_to_memset), size - size_to_memset)
+                .context("failed to decommit table page")?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::InstanceLimits;
+
+    #[cfg(target_pointer_width = "64")]
+    #[test]
+    fn test_table_pool() -> Result<()> {
+        let pool = TablePool::new(&PoolingInstanceAllocatorConfig {
+            limits: InstanceLimits {
+                total_tables: 7,
+                table_elements: 100,
+                memory_pages: 0,
+                max_memories_per_module: 0,
+                ..Default::default()
+            },
+            ..Default::default()
+        })?;
+
+        let host_page_size = crate::page_size();
+
+        assert_eq!(pool.table_size, host_page_size);
+        assert_eq!(pool.max_total_tables, 7);
+        assert_eq!(pool.page_size, host_page_size);
+        assert_eq!(pool.table_elements, 100);
+
+        let base = pool.mapping.as_ptr() as usize;
+
+        for i in 0..7 {
+            let index = TableAllocationIndex(i);
+            let ptr = pool.get(index);
+            assert_eq!(ptr as usize - base, i as usize * pool.table_size);
+        }
+
+        Ok(())
+    }
+}