wasmtime 17.0.1 → 18.0.3

data/ext/cargo-vendor/winch-codegen-0.16.3/src/codegen/env.rs

@@ -0,0 +1,257 @@
+use crate::codegen::{control, BlockSig, BuiltinFunction, OperandSize};
+use std::collections::{
+    hash_map::Entry::{Occupied, Vacant},
+    HashMap,
+};
+use wasmparser::BlockType;
+use wasmtime_environ::{
+    FuncIndex, GlobalIndex, MemoryIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize,
+    TableIndex, TablePlan, TypeConvert, TypeIndex, VMOffsets, WasmFuncType, WasmHeapType,
+    WasmValType,
+};
+
+/// Table metadata.
+#[derive(Debug, Copy, Clone)]
+pub struct TableData {
+    /// The offset to the base of the table.
+    pub offset: u32,
+    /// The offset to the current elements field.
+    pub current_elems_offset: u32,
+    /// If the table is imported, this field contains the offset to locate the
+    /// base of the table data.
+    pub import_from: Option<u32>,
+    /// The size of the table elements.
+    pub(crate) element_size: OperandSize,
+    /// The size of the current elements field.
+    pub(crate) current_elements_size: OperandSize,
+}
+
+/// Heap metadata.
+#[derive(Debug, Copy, Clone)]
+pub struct HeapData {
+    /// The offset to the base of the heap.
+    pub offset: u32,
+    /// The offset to the current length field.
+    pub current_length_offset: u32,
+    /// If the heap is imported, this field contains the offset to locate the
+    /// base of the heap.
+    pub import_from: Option<u32>,
+    /// The memory type (32 or 64).
+    pub ty: WasmValType,
+}
+
+/// A function callee.
+/// It categorizes how the callee should be treated
+/// when performing the call.
+#[derive(Clone)]
+pub enum Callee {
+    /// Locally defined function.
+    Local(CalleeInfo),
+    /// Imported function.
+    Import(CalleeInfo),
+    /// Function reference.
+    FuncRef(WasmFuncType),
+    /// A built-in function.
+    Builtin(BuiltinFunction),
+}
+
+/// Metadata about a function callee. Used by the code generation to
+/// emit function calls to local or imported functions.
+#[derive(Clone)]
+pub struct CalleeInfo {
+    /// The function type.
+    pub ty: WasmFuncType,
+    /// The callee index in the WebAssembly function index space.
+    pub index: FuncIndex,
+}
+
+/// The function environment.
+///
+/// Contains all information about the module and runtime that is accessible to
+/// to a particular function during code generation.
+pub struct FuncEnv<'a, 'translation: 'a, 'data: 'translation, P: PtrSize> {
+    /// Offsets to the fields within the `VMContext` ptr.
+    pub vmoffsets: &'a VMOffsets<P>,
+    /// Metadata about the translation process of a WebAssembly module.
+    pub translation: &'translation ModuleTranslation<'data>,
+    /// The module's function types.
+    pub types: &'translation ModuleTypesBuilder,
+    /// Track resolved table information.
+    resolved_tables: HashMap<TableIndex, TableData>,
+    resolved_heaps: HashMap<MemoryIndex, HeapData>,
+}
+
+pub fn ptr_type_from_ptr_size(size: u8) -> WasmValType {
+    (size == 8)
+        .then(|| WasmValType::I64)
+        .unwrap_or_else(|| unimplemented!("Support for non-64-bit architectures"))
+}
+
+impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
+    /// Create a new function environment.
+    pub fn new(
+        vmoffsets: &'a VMOffsets<P>,
+        translation: &'translation ModuleTranslation<'data>,
+        types: &'translation ModuleTypesBuilder,
+    ) -> Self {
+        Self {
+            vmoffsets,
+            translation,
+            types,
+            resolved_tables: HashMap::new(),
+            resolved_heaps: HashMap::new(),
+        }
+    }
+
+    /// Derive the [`WasmType`] from the pointer size.
+    pub(crate) fn ptr_type(&self) -> WasmValType {
+        ptr_type_from_ptr_size(self.ptr_size())
+    }
+
+    /// Returns the pointer size for the target ISA.
+    fn ptr_size(&self) -> u8 {
+        self.vmoffsets.ptr.size()
+    }
+
+    /// Resolves a [`Callee::FuncRef`] from a type index.
+    pub fn funcref(&self, idx: TypeIndex) -> Callee {
+        let sig_index = self.translation.module.types[idx].unwrap_function();
+        let ty = self.types[sig_index].clone();
+        Callee::FuncRef(ty)
+    }
+
+    /// Resolves a function [`Callee`] from an index.
+    pub fn callee_from_index(&self, idx: FuncIndex) -> Callee {
+        let types = &self.translation.get_types();
+        let ty = types[types.core_function_at(idx.as_u32())].unwrap_func();
+        let ty = self.convert_func_type(ty);
+        let import = self.translation.module.is_imported_function(idx);
+
+        let info = CalleeInfo { ty, index: idx };
+
+        if import {
+            Callee::Import(info)
+        } else {
+            Callee::Local(info)
+        }
+    }
+
+    /// Converts a [wasmparser::BlockType] into a [BlockSig].
+    pub(crate) fn resolve_block_sig(&self, ty: BlockType) -> BlockSig {
+        use BlockType::*;
+        match ty {
+            Empty => BlockSig::new(control::BlockType::void()),
+            Type(ty) => {
+                let ty = self.convert_valtype(ty);
+                BlockSig::new(control::BlockType::single(ty))
+            }
+            FuncType(idx) => {
+                let sig_index =
+                    self.translation.module.types[TypeIndex::from_u32(idx)].unwrap_function();
+                let sig = &self.types[sig_index];
+                BlockSig::new(control::BlockType::func(sig.clone()))
+            }
+        }
+    }
+
+    /// Resolves the type and offset of a global at the given index.
+    pub fn resolve_global_type_and_offset(&self, index: GlobalIndex) -> (WasmValType, u32) {
+        let ty = self.translation.module.globals[index].wasm_ty;
+        let offset = match self.translation.module.defined_global_index(index) {
+            Some(defined_index) => self.vmoffsets.vmctx_vmglobal_definition(defined_index),
+            None => self.vmoffsets.vmctx_vmglobal_import_from(index),
+        };
+
+        (ty, offset)
+    }
+
+    /// Returns the table information for the given table index.
+    pub fn resolve_table_data(&mut self, index: TableIndex) -> TableData {
+        match self.resolved_tables.entry(index) {
+            Occupied(entry) => *entry.get(),
+            Vacant(entry) => {
+                let (from_offset, base_offset, current_elems_offset) =
+                    match self.translation.module.defined_table_index(index) {
+                        Some(defined) => (
+                            None,
+                            self.vmoffsets.vmctx_vmtable_definition_base(defined),
+                            self.vmoffsets
+                                .vmctx_vmtable_definition_current_elements(defined),
+                        ),
+                        None => (
+                            Some(self.vmoffsets.vmctx_vmtable_import_from(index)),
+                            self.vmoffsets.vmtable_definition_base().into(),
+                            self.vmoffsets.vmtable_definition_current_elements().into(),
+                        ),
+                    };
+
+                *entry.insert(TableData {
+                    import_from: from_offset,
+                    offset: base_offset,
+                    current_elems_offset,
+                    element_size: OperandSize::from_bytes(self.vmoffsets.ptr.size()),
+                    current_elements_size: OperandSize::from_bytes(
+                        self.vmoffsets.size_of_vmtable_definition_current_elements(),
+                    ),
+                })
+            }
+        }
+    }
+
+    /// Resolved a [HeapData] from a [MemoryIndex].
+    // TODO: (@saulecabrera)
+    // Handle shared memories when implementing support for Wasm Threads.
+    pub fn resolve_heap(&mut self, index: MemoryIndex) -> HeapData {
+        match self.resolved_heaps.entry(index) {
+            Occupied(entry) => *entry.get(),
+            Vacant(entry) => {
+                let (import_from, base_offset, current_length_offset) =
+                    match self.translation.module.defined_memory_index(index) {
+                        Some(defined) => {
+                            let owned = self.translation.module.owned_memory_index(defined);
+                            (
+                                None,
+                                self.vmoffsets.vmctx_vmmemory_definition_base(owned),
+                                self.vmoffsets
+                                    .vmctx_vmmemory_definition_current_length(owned),
+                            )
+                        }
+                        None => (
+                            Some(self.vmoffsets.vmctx_vmmemory_import_from(index)),
+                            self.vmoffsets.ptr.vmmemory_definition_base().into(),
+                            self.vmoffsets
+                                .ptr
+                                .vmmemory_definition_current_length()
+                                .into(),
+                        ),
+                    };
+
+                *entry.insert(HeapData {
+                    offset: base_offset,
+                    import_from,
+                    current_length_offset,
+                    ty: if self.translation.module.memory_plans[index].memory.memory64 {
+                        WasmValType::I64
+                    } else {
+                        WasmValType::I32
+                    },
+                })
+            }
+        }
+    }
+
+    /// Get a [`TablePlan`] from a [`TableIndex`].
+    pub fn table_plan(&mut self, index: TableIndex) -> &TablePlan {
+        &self.translation.module.table_plans[index]
+    }
+}
+
+impl<P: PtrSize> TypeConvert for FuncEnv<'_, '_, '_, P> {
+    fn lookup_heap_type(&self, idx: wasmparser::UnpackedIndex) -> WasmHeapType {
+        wasmtime_environ::WasmparserTypeConverter {
+            module: &self.translation.module,
+            types: self.types,
+        }
+        .lookup_heap_type(idx)
+    }
+}

data/ext/cargo-vendor/winch-codegen-0.16.3/src/codegen/mod.rs

@@ -0,0 +1,441 @@
+use crate::{
+    abi::{ABIOperand, ABISig, RetArea, ABI},
+    codegen::BlockSig,
+    isa::reg::Reg,
+    masm::{IntCmpKind, MacroAssembler, OperandSize, RegImm, SPOffset, TrapCode},
+    stack::TypedReg,
+};
+use anyhow::Result;
+use smallvec::SmallVec;
+use wasmparser::{BinaryReader, FuncValidator, Operator, ValidatorResources, VisitOperator};
+use wasmtime_environ::{PtrSize, TableIndex, TypeIndex, WasmHeapType, WasmValType, FUNCREF_MASK};
+
+mod context;
+pub(crate) use context::*;
+mod env;
+pub use env::*;
+mod call;
+pub(crate) use call::*;
+mod control;
+pub(crate) use control::*;
+mod builtin;
+pub use builtin::*;
+
+/// The code generation abstraction.
+pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M>
+where
+    M: MacroAssembler,
+{
+    /// The ABI-specific representation of the function signature, excluding results.
+    pub sig: ABISig,
+
+    /// The code generation context.
+    pub context: CodeGenContext<'a, 'translation>,
+
+    /// A reference to the function compilation environment.
+    pub env: FuncEnv<'a, 'translation, 'data, M::Ptr>,
+
+    /// The MacroAssembler.
+    pub masm: &'a mut M,
+
+    /// Stack frames for control flow.
+    // NB The 64 is set arbitrarily, we can adjust it as
+    // we see fit.
+    pub control_frames: SmallVec<[ControlStackFrame; 64]>,
+}
+
+impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M>
+where
+    M: MacroAssembler,
+{
+    pub fn new(
+        masm: &'a mut M,
+        context: CodeGenContext<'a, 'translation>,
+        env: FuncEnv<'a, 'translation, 'data, M::Ptr>,
+        sig: ABISig,
+    ) -> Self {
+        Self {
+            sig,
+            context,
+            masm,
+            env,
+            control_frames: Default::default(),
+        }
+    }
+
+    /// Emit the function body to machine code.
+    pub fn emit(
+        &mut self,
+        body: &mut BinaryReader<'a>,
+        validator: &mut FuncValidator<ValidatorResources>,
+    ) -> Result<()> {
+        self.emit_start()
+            .and_then(|_| self.emit_body(body, validator))
+            .and_then(|_| self.emit_end())?;
+
+        Ok(())
+    }
+
+    fn emit_start(&mut self) -> Result<()> {
+        self.masm.prologue();
+        self.masm.reserve_stack(self.context.frame.locals_size);
+
+        // Check for stack overflow after reserving space, so that we get the most up-to-date view
+        // of the stack pointer. This assumes that no writes to the stack occur in `reserve_stack`.
+        self.masm.check_stack();
+
+        // Once we have emitted the epilogue and reserved stack space for the locals, we push the
+        // base control flow block.
+        self.control_frames.push(ControlStackFrame::block(
+            BlockSig::from_sig(self.sig.clone()),
+            self.masm,
+            &mut self.context,
+        ));
+
+        // Set the return area of the results *after* initializing the block. In
+        // the function body block case, we'll treat the results as any other
+        // case, addressed from the stack pointer, and when ending the function
+        // the return area will be set to the return pointer.
+        if self.sig.params.has_retptr() {
+            self.sig
+                .results
+                .set_ret_area(RetArea::slot(self.context.frame.results_base_slot.unwrap()));
+        }
+
+        Ok(())
+    }
+
+    /// The following two helpers, handle else or end instructions when the
+    /// compiler has entered into an unreachable code state. These instructions
+    /// must be observed to determine if the reachability state should be
+    /// restored.
+    ///
+    /// When the compiler is in an unreachable state, all the other instructions
+    /// are not visited.
+    pub fn handle_unreachable_else(&mut self) {
+        let frame = self.control_frames.last_mut().unwrap();
+        debug_assert!(frame.is_if());
+        if frame.is_next_sequence_reachable() {
+            // We entered an unreachable state when compiling the
+            // if-then branch, but if the `if` was reachable at
+            // entry, the if-else branch will be reachable.
+            self.context.reachable = true;
+            frame.ensure_stack_state(self.masm, &mut self.context);
+            frame.bind_else(self.masm, &mut self.context);
+        }
+    }
+
+    pub fn handle_unreachable_end(&mut self) {
+        let mut frame = self.control_frames.pop().unwrap();
+        // We just popped the outermost block.
+        let is_outermost = self.control_frames.len() == 0;
+
+        if frame.is_next_sequence_reachable() {
+            self.context.reachable = true;
+            frame.ensure_stack_state(self.masm, &mut self.context);
+            frame.bind_end(self.masm, &mut self.context);
+        } else if is_outermost {
+            // If we reach the end of the function in an unreachable
+            // state, perform the necessary cleanup to leave the stack
+            // and SP in the expected state.  The compiler can enter
+            // in this state through an infinite loop.
+            frame.ensure_stack_state(self.masm, &mut self.context);
+        }
+    }
+
+    fn emit_body(
+        &mut self,
+        body: &mut BinaryReader<'a>,
+        validator: &mut FuncValidator<ValidatorResources>,
+    ) -> Result<()> {
+        self.spill_register_arguments();
+        let defined_locals_range = &self.context.frame.defined_locals_range;
+        self.masm.zero_mem_range(defined_locals_range.as_range());
+
+        // Save the vmctx pointer to its local slot in case we need to reload it
+        // at any point.
+        let vmctx_addr = self.masm.local_address(&self.context.frame.vmctx_slot);
+        self.masm
+            .store_ptr(<M::ABI as ABI>::vmctx_reg().into(), vmctx_addr);
+
+        // Save the results base parameter register into its slot.
+        self.sig.params.has_retptr().then(|| {
+            match self.sig.params.unwrap_results_area_operand() {
+                ABIOperand::Reg { ty, reg, .. } => {
+                    let results_base_slot = self.context.frame.results_base_slot.as_ref().unwrap();
+                    debug_assert!(results_base_slot.addressed_from_sp());
+                    let addr = self.masm.local_address(results_base_slot);
+                    self.masm.store((*reg).into(), addr, (*ty).into());
+                }
+                // The result base parameter is a stack paramter, addressed
+                // from FP.
+                _ => {}
+            }
+        });
+
+        while !body.eof() {
+            let offset = body.original_position();
+            body.visit_operator(&mut ValidateThenVisit(validator.visitor(offset), self))??;
+        }
+        validator.finish(body.original_position())?;
+        return Ok(());
+
+        struct ValidateThenVisit<'a, T, U>(T, &'a mut U);
+
+        macro_rules! validate_then_visit {
+            ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident)*) => {
+                $(
+                    fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output {
+                        self.0.$visit($($($arg.clone()),*)?)?;
+                        // Only visit operators if the compiler is in a reachable code state. If
+                        // the compiler is in an unrechable code state, most of the operators are
+                        // ignored except for If, Block, Loop, Else and End. These operators need
+                        // to be observed in order to keep the control stack frames balanced and to
+                        // determine if reachability should be restored.
+                        let visit_when_unreachable = visit_op_when_unreachable(Operator::$op $({ $($arg: $arg.clone()),* })?);
+                        if self.1.is_reachable() || visit_when_unreachable  {
+                            Ok(self.1.$visit($($($arg),*)?))
+                        } else {
+                            Ok(U::Output::default())
+                        }
+                    }
+                )*
+            };
+        }
+
+        fn visit_op_when_unreachable(op: Operator) -> bool {
+            use Operator::*;
+            match op {
+                If { .. } | Block { .. } | Loop { .. } | Else | End => true,
+                _ => false,
+            }
+        }
+
+        /// Trait to handle reachability state.
+        trait ReachableState {
+            /// Returns true if the current state of the program is reachable.
+            fn is_reachable(&self) -> bool;
+        }
+
+        impl<'a, 'translation, 'data, M: MacroAssembler> ReachableState
+            for CodeGen<'a, 'translation, 'data, M>
+        {
+            fn is_reachable(&self) -> bool {
+                self.context.reachable
+            }
+        }
+
+        impl<'a, T, U> VisitOperator<'a> for ValidateThenVisit<'_, T, U>
+        where
+            T: VisitOperator<'a, Output = wasmparser::Result<()>>,
+            U: VisitOperator<'a> + ReachableState,
+            U::Output: Default,
+        {
+            type Output = Result<U::Output>;
+
+            wasmparser::for_each_operator!(validate_then_visit);
+        }
+    }
+
+    /// Emits a a series of instructions that will type check a function reference call.
+    pub fn emit_typecheck_funcref(&mut self, funcref_ptr: Reg, type_index: TypeIndex) {
+        let ptr_size: OperandSize = self.env.ptr_type().into();
+        let sig_index_bytes = self.env.vmoffsets.size_of_vmshared_type_index();
+        let sig_size = OperandSize::from_bytes(sig_index_bytes);
+        let sig_index = self.env.translation.module.types[type_index].unwrap_function();
+        let sig_offset = sig_index
+            .as_u32()
+            .checked_mul(sig_index_bytes.into())
+            .unwrap();
+        let signatures_base_offset = self.env.vmoffsets.vmctx_type_ids_array();
+        let scratch = <M::ABI as ABI>::scratch_reg();
+        let funcref_sig_offset = self.env.vmoffsets.ptr.vm_func_ref_type_index();
+
+        // Load the signatures address into the scratch register.
+        self.masm.load(
+            self.masm.address_at_vmctx(signatures_base_offset),
+            scratch,
+            ptr_size,
+        );
+
+        // Get the caller id.
+        let caller_id = self.context.any_gpr(self.masm);
+        self.masm.load(
+            self.masm.address_at_reg(scratch, sig_offset),
+            caller_id,
+            sig_size,
+        );
+
+        let callee_id = self.context.any_gpr(self.masm);
+        self.masm.load(
+            self.masm
+                .address_at_reg(funcref_ptr, funcref_sig_offset.into()),
+            callee_id,
+            sig_size,
+        );
+
+        // Typecheck.
+        self.masm.cmp(callee_id.into(), caller_id, OperandSize::S32);
+        self.masm.trapif(IntCmpKind::Ne, TrapCode::BadSignature);
+        self.context.free_reg(callee_id);
+        self.context.free_reg(caller_id);
+    }
+
+    /// Emit the usual function end instruction sequence.
+    fn emit_end(&mut self) -> Result<()> {
+        // The implicit body block is treated a normal block (it pushes results
+        // to the stack); so when reaching the end, we pop them taking as
+        // reference the current function's signature.
+        let base = SPOffset::from_u32(self.context.frame.locals_size);
+        if self.context.reachable {
+            ControlStackFrame::pop_abi_results_impl(
+                &mut self.sig.results,
+                &mut self.context,
+                self.masm,
+                |results, _, _| results.ret_area().copied(),
+            );
+        } else {
+            // If we reach the end of the function in a unreachable code state,
+            // simly truncate to the the expected values.
+            // The compiler could enter in this state through an infinite loop.
+            self.context.truncate_stack_to(0);
+            self.masm.reset_stack_pointer(base);
+        }
+        debug_assert_eq!(self.context.stack.len(), 0);
+        self.masm.epilogue(self.context.frame.locals_size);
+        Ok(())
+    }
+
+    fn spill_register_arguments(&mut self) {
+        use WasmValType::*;
+        self.sig
+            // Skip the results base param if any; [Self::emit_body],
+            // will handle spilling the results base param if it's in a register.
+            .params_without_retptr()
+            .iter()
+            .enumerate()
+            .filter(|(_, a)| a.is_reg())
+            .for_each(|(index, arg)| {
+                let ty = arg.ty();
+                let local = self
+                    .context
+                    .frame
+                    .get_local(index as u32)
+                    .expect("valid local slot at location");
+                let addr = self.masm.local_address(local);
+                let src = arg
+                    .get_reg()
+                    .expect("arg should be associated to a register");
+
+                match &ty {
+                    I32 | I64 | F32 | F64 => self.masm.store(src.into(), addr, ty.into()),
+                    Ref(rt) => match rt.heap_type {
+                        WasmHeapType::Func => self.masm.store_ptr(src.into(), addr),
+                        ht => unimplemented!("Support for WasmHeapType: {ht}"),
+                    },
+                    _ => unimplemented!("Support for WasmType {ty}"),
+                }
+            });
+    }
+
+    /// Pops the value at the stack top and assigns it to the local at
+    /// the given index, returning the typed register holding the
+    /// source value.
+    pub fn emit_set_local(&mut self, index: u32) -> TypedReg {
+        // Materialize any references to the same local index that are in the
+        // value stack by spilling.
+        if self.context.stack.contains_latent_local(index) {
+            self.context.spill(self.masm);
+        }
+        let src = self.context.pop_to_reg(self.masm, None);
+        // Need to get address of local after `pop_to_reg` since `pop_to_reg`
+        // will pop the machine stack causing an incorrect address to be
+        // calculated.
+        let (ty, addr) = self.context.frame.get_local_address(index, self.masm);
+        self.masm.store(RegImm::reg(src.reg), addr, ty.into());
+
+        src
+    }
+
+    pub fn emit_lazy_init_funcref(&mut self, table_index: TableIndex) {
+        let table_data = self.env.resolve_table_data(table_index);
+        let ptr_type = self.env.ptr_type();
+        let builtin = self
+            .context
+            .builtins
+            .table_get_lazy_init_func_ref::<M::ABI, M::Ptr>();
+
+        // Request the builtin's  result register and use it to hold the
+        // table element value. We preemptively request this register to
+        // avoid conflict at the control flow merge below.
+        // Requesting the result register is safe since we know ahead-of-time
+        // the builtin's signature.
+        let elem_value: Reg = self
+            .context
+            .reg(
+                builtin.sig().results.unwrap_singleton().unwrap_reg(),
+                self.masm,
+            )
+            .into();
+
+        let index = self.context.pop_to_reg(self.masm, None);
+        let base = self.context.any_gpr(self.masm);
+
+        let elem_addr =
+            self.masm
+                .table_elem_address(index.into(), base, &table_data, &mut self.context);
+        self.masm.load_ptr(elem_addr, elem_value);
+        // Free the register used as base, once we have loaded the element
+        // address into the element value register.
+        self.context.free_reg(base);
+
+        let (defined, cont) = (self.masm.get_label(), self.masm.get_label());
+
+        // Push the built-in arguments to the stack.
+        self.context.stack.extend([
+            TypedReg::new(ptr_type, <M::ABI as ABI>::vmctx_reg()).into(),
+            table_index.as_u32().try_into().unwrap(),
+            index.into(),
+        ]);
+
+        self.masm.branch(
+            IntCmpKind::Ne,
+            elem_value.into(),
+            elem_value,
+            defined,
+            ptr_type.into(),
+        );
+        // Free the element value register.
+        // This is safe since the FnCall::emit call below, will ensure
+        // that the result register is placed on the value stack.
+        self.context.free_reg(elem_value);
+        FnCall::emit::<M, M::Ptr, _>(self.masm, &mut self.context, |_| {
+            Callee::Builtin(builtin.clone())
+        });
+
+        // We know the signature of the libcall in this case, so we assert that there's
+        // one element in the stack and that it's  the ABI signature's result register.
+        let top = self.context.stack.peek().unwrap();
+        let top = top.unwrap_reg();
+        debug_assert!(top.reg == elem_value);
+        self.masm.jmp(cont);
+
+        // In the defined case, mask the funcref address in place, by peeking into the
+        // last element of the value stack, which was pushed by the `indirect` function
+        // call above.
+        self.masm.bind(defined);
+        let imm = RegImm::i64(FUNCREF_MASK as i64);
+        let dst = top.into();
+        self.masm.and(dst, dst, imm, top.ty.into());
+
+        self.masm.bind(cont);
+    }
+}
+
+/// Returns the index of the [`ControlStackFrame`] for the given
+/// depth.
+pub fn control_index(depth: u32, control_length: usize) -> usize {
+    (control_length - 1)
+        .checked_sub(depth as usize)
+        .unwrap_or_else(|| panic!("expected valid control stack frame at index: {}", depth))
+}