wasmtime 15.0.1 → 16.0.0

data/ext/cargo-vendor/winch-codegen-0.14.0/src/trampoline.rs

@@ -0,0 +1,734 @@
+//! Trampoline implementation for Winch.
+//!
+//! This module contains all the necessary pieces to emit the various
+//! trampolines required by Wasmtime to call JIT code.
+//
+// TODO
+//
+// * Remove the usage of hardcoded operand sizes (`OperandSize::S64`) when
+// loading/storing the VM context pointer. The real value of the operand size
+// and VM context type should be derived from the ABI's pointer size. This is
+// going to be relevant once 32-bit architectures are supported.
+//
+// TODO: Are guardrails needed for params/results? Especially when dealing
+// with the array calling convention.
+use crate::{
+    abi::{ABIOperand, ABIParams, ABISig, RetArea, ABI},
+    codegen::ptr_type_from_ptr_size,
+    isa::CallingConvention,
+    masm::{CalleeKind, MacroAssembler, OperandSize, RegImm, SPOffset},
+    reg::Reg,
+};
+use anyhow::{anyhow, Result};
+use smallvec::SmallVec;
+use std::mem;
+use wasmtime_environ::{FuncIndex, PtrSize, WasmFuncType, WasmType};
+
+/// The supported trampoline kinds.
+/// See <https://github.com/bytecodealliance/rfcs/blob/main/accepted/tail-calls.md#new-trampolines-and-vmcallercheckedanyfunc-changes>
+/// for more details.
+pub enum TrampolineKind {
+    /// Calling from native to Wasm, using the array calling convention.
+    ArrayToWasm(FuncIndex),
+    /// Calling from native to Wasm.
+    NativeToWasm(FuncIndex),
+    /// Calling from Wasm to native.
+    WasmToNative,
+}
+
+/// The max value size of an element in the array calling convention.
+const VALUE_SIZE: usize = mem::size_of::<u128>();
+
+/// The main trampoline abstraction.
+pub(crate) struct Trampoline<'a, M>
+where
+    M: MacroAssembler,
+{
+    /// The macro assembler.
+    masm: &'a mut M,
+    /// The main scratch register for the current architecture. It is
+    /// not allocatable for the callee.
+    scratch_reg: Reg,
+    /// A second scratch register. This will be allocatable for the
+    /// callee, so it can only be used after the callee-saved
+    /// registers are on the stack.
+    alloc_scratch_reg: Reg,
+    /// Registers to be saved as part of the trampoline's prologue
+    /// and to be restored as part of the trampoline's epilogue.
+    callee_saved_regs: SmallVec<[(Reg, OperandSize); 18]>,
+    /// The calling convention used by the trampoline,
+    /// which is the Wasmtime variant of the system ABI's
+    /// calling convention.
+    call_conv: &'a CallingConvention,
+    /// The pointer size of the current ISA.
+    pointer_size: M::Ptr,
+    /// WasmType representation of the pointer size.
+    pointer_type: WasmType,
+}
+
+impl<'a, M> Trampoline<'a, M>
+where
+    M: MacroAssembler,
+{
+    /// Create a new trampoline.
+    pub fn new(
+        masm: &'a mut M,
+        scratch_reg: Reg,
+        alloc_scratch_reg: Reg,
+        call_conv: &'a CallingConvention,
+        pointer_size: M::Ptr,
+    ) -> Self {
+        let size = pointer_size.size();
+        Self {
+            masm,
+            scratch_reg,
+            alloc_scratch_reg,
+            callee_saved_regs: <M::ABI as ABI>::callee_saved_regs(call_conv),
+            call_conv,
+            pointer_size,
+            pointer_type: ptr_type_from_ptr_size(size),
+        }
+    }
+
+    /// Emit an array-to-wasm trampoline.
+    pub fn emit_array_to_wasm(&mut self, ty: &WasmFuncType, callee_index: FuncIndex) -> Result<()> {
+        let array_sig = self.array_sig();
+        let wasm_sig = self.wasm_sig(ty);
+
+        let val_ptr = array_sig
+            .params
+            .get(2)
+            .map(|operand| RegImm::reg(operand.unwrap_reg()))
+            .ok_or_else(|| anyhow!("Expected value pointer to be in a register"))?;
+
+        self.prologue_with_callee_saved();
+
+        // Get the VM context pointer and move it to the designated pinned
+        // register.
+        let (vmctx, caller_vmctx) = Self::callee_and_caller_vmctx(&array_sig.params)?;
+
+        self.masm.mov(
+            vmctx.into(),
+            <M::ABI as ABI>::vmctx_reg().into(),
+            OperandSize::S64,
+        );
+
+        let ret_area = self.make_ret_area(&wasm_sig);
+        let vmctx_runtime_limits_addr = self.vmctx_runtime_limits_addr(caller_vmctx);
+        let (offsets, spill_size) = self.spill(array_sig.params());
+
+        // Call the function that was passed into the trampoline.
+        let allocated_stack = self.masm.call(wasm_sig.params_stack_size(), |masm| {
+            // Save the SP when entering Wasm.
+            // TODO: Once Winch supports comparison operators,
+            // check that the caller VM context is what we expect.
+            // See [`wasmtime_environ::MAGIC`].
+            Self::save_last_wasm_entry_sp(
+                masm,
+                vmctx_runtime_limits_addr,
+                self.scratch_reg,
+                &self.pointer_size,
+            );
+
+            // Move the values register to the scratch
+            // register for argument assignment.
+            masm.mov(val_ptr, self.scratch_reg.into(), OperandSize::S64);
+            Self::load_values_from_array(
+                masm,
+                &wasm_sig,
+                ret_area.as_ref(),
+                self.scratch_reg,
+                self.alloc_scratch_reg,
+            );
+            CalleeKind::Direct(callee_index.as_u32())
+        });
+
+        self.masm.free_stack(allocated_stack);
+
+        // Move the val ptr back into the scratch register so we can
+        // load the return values.
+        let val_ptr_offset = offsets[2];
+        self.masm.load(
+            self.masm.address_from_sp(val_ptr_offset),
+            self.scratch_reg,
+            OperandSize::S64,
+        );
+
+        self.store_results_to_array(&wasm_sig, ret_area.as_ref());
+
+        if wasm_sig.results.has_stack_results() {
+            self.masm.free_stack(wasm_sig.results.size());
+        }
+
+        self.epilogue_with_callee_saved_restore(spill_size);
+        Ok(())
+    }
+
+    /// Stores the results into the values array used by the array calling
+    /// convention.
+    fn store_results_to_array(&mut self, sig: &ABISig, ret_area: Option<&RetArea>) {
+        for (i, operand) in sig.results().iter().enumerate() {
+            let value_offset = (i * VALUE_SIZE) as u32;
+            match operand {
+                ABIOperand::Reg { ty, reg, .. } => self.masm.store(
+                    (*reg).into(),
+                    self.masm.address_at_reg(self.scratch_reg, value_offset),
+                    (*ty).into(),
+                ),
+                ABIOperand::Stack { ty, offset, .. } => {
+                    let addr = match ret_area.unwrap() {
+                        RetArea::SP(sp_offset) => {
+                            let elem_offs = SPOffset::from_u32(sp_offset.as_u32() - offset);
+                            self.masm.address_from_sp(elem_offs)
+                        }
+                        _ => unreachable!(),
+                    };
+                    self.masm.load(addr, self.alloc_scratch_reg, (*ty).into());
+                    self.masm.store(
+                        self.alloc_scratch_reg.into(),
+                        self.masm.address_at_reg(self.scratch_reg, value_offset),
+                        (*ty).into(),
+                    );
+                }
+            }
+        }
+    }
+
+    /// Emit a native-to-wasm trampoline.
+    pub fn emit_native_to_wasm(
+        &mut self,
+        ty: &WasmFuncType,
+        callee_index: FuncIndex,
+    ) -> Result<()> {
+        let native_sig = self.native_sig(&ty);
+        let wasm_sig = self.wasm_sig(&ty);
+        let (vmctx, caller_vmctx) = Self::callee_and_caller_vmctx(&native_sig.params)?;
+
+        self.prologue_with_callee_saved();
+        // Move the VM context pointer to the designated pinned register.
+        self.masm.mov(
+            vmctx.into(),
+            <M::ABI as ABI>::vmctx_reg().into(),
+            OperandSize::S64,
+        );
+
+        let vmctx_runtime_limits_addr = self.vmctx_runtime_limits_addr(caller_vmctx);
+        let ret_area = self.make_ret_area(&wasm_sig);
+        let (offsets, spill_size) = self.spill(native_sig.params());
+
+        let reserved_stack = self.masm.call(wasm_sig.params_stack_size(), |masm| {
+            // Save the SP when entering Wasm.
+            // TODO: Once Winch supports comparison operators,
+            // check that the caller VM context is what we expect.
+            // See [`wasmtime_environ::MAGIC`].
+            Self::save_last_wasm_entry_sp(
+                masm,
+                vmctx_runtime_limits_addr,
+                self.scratch_reg,
+                &self.pointer_size,
+            );
+            Self::assign_args(
+                masm,
+                &wasm_sig.params_without_retptr(),
+                &native_sig.params_without_retptr()[2..],
+                &offsets[2..],
+                self.scratch_reg,
+            );
+            Self::load_retptr(masm, ret_area.as_ref(), &wasm_sig);
+            CalleeKind::Direct(callee_index.as_u32())
+        });
+
+        self.masm.free_stack(reserved_stack);
+        self.forward_results(&wasm_sig, &native_sig, ret_area.as_ref(), offsets.last());
+        if wasm_sig.results.has_stack_results() {
+            self.masm.free_stack(wasm_sig.results.size());
+        }
+        self.epilogue_with_callee_saved_restore(spill_size);
+
+        Ok(())
+    }
+
+    /// Creates the return area in the caller's frame.
+    fn make_ret_area(&mut self, sig: &ABISig) -> Option<RetArea> {
+        sig.results.has_stack_results().then(|| {
+            self.masm.reserve_stack(sig.results.size());
+            let offs = self.masm.sp_offset();
+            RetArea::sp(offs)
+        })
+    }
+
+    /// Loads the return area pointer into its [ABIOperand] destination.
+    fn load_retptr(masm: &mut M, ret_area: Option<&RetArea>, callee: &ABISig) {
+        if let Some(area) = ret_area {
+            match (area, callee.params.unwrap_results_area_operand()) {
+                (RetArea::SP(sp_offset), ABIOperand::Reg { ty, reg, .. }) => {
+                    let addr = masm.address_from_sp(*sp_offset);
+                    masm.load_addr(addr, *reg, (*ty).into());
+                }
+                (RetArea::SP(sp_offset), ABIOperand::Stack { ty, offset, .. }) => {
+                    let retptr = masm.address_from_sp(*sp_offset);
+                    let scratch = <M::ABI as ABI>::scratch_reg();
+                    masm.load_addr(retptr, scratch, (*ty).into());
+                    let retptr_slot = masm.address_from_sp(SPOffset::from_u32(*offset));
+                    masm.store(scratch.into(), retptr_slot, (*ty).into());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    /// Forwards results from callee to caller; it loads results from the
+    /// callee's return area and stores them into the caller's return area.
+    fn forward_results(
+        &mut self,
+        callee_sig: &ABISig,
+        caller_sig: &ABISig,
+        callee_ret_area: Option<&RetArea>,
+        caller_retptr_offset: Option<&SPOffset>,
+    ) {
+        // Spill any result registers used by the callee to avoid
+        // use-assign issues when forwarding the results.
+        let results_spill = self.spill(callee_sig.results());
+        let mut spill_offsets_iter = results_spill.0.iter();
+
+        let caller_retptr = caller_sig.results.has_stack_results().then(|| {
+            let fp = <M::ABI as ABI>::fp_reg();
+            let arg_base: u32 = <M::ABI as ABI>::arg_base_offset().into();
+            match caller_sig.params.unwrap_results_area_operand() {
+                ABIOperand::Reg { ty, .. } => {
+                    let addr = self.masm.address_from_sp(*caller_retptr_offset.unwrap());
+                    self.masm.load(addr, self.scratch_reg, (*ty).into());
+                    self.scratch_reg
+                }
+                ABIOperand::Stack { ty, offset, .. } => {
+                    let addr = self.masm.address_at_reg(fp, arg_base + offset);
+                    self.masm.load(addr, self.scratch_reg, (*ty).into());
+                    self.scratch_reg
+                }
+            }
+        });
+
+        for (callee_operand, caller_operand) in
+            callee_sig.results().iter().zip(caller_sig.results())
+        {
+            match (callee_operand, caller_operand) {
+                (ABIOperand::Reg { ty, .. }, ABIOperand::Stack { offset, .. }) => {
+                    let reg_offset = spill_offsets_iter.next().unwrap();
+                    self.masm.load(
+                        self.masm.address_from_sp(*reg_offset),
+                        self.alloc_scratch_reg,
+                        (*ty).into(),
+                    );
+                    self.masm.store(
+                        self.alloc_scratch_reg.into(),
+                        self.masm.address_at_reg(caller_retptr.unwrap(), *offset),
+                        (*ty).into(),
+                    );
+                }
+                (
+                    ABIOperand::Stack { ty, offset, .. },
+                    ABIOperand::Stack {
+                        offset: caller_offset,
+                        ..
+                    },
+                ) => {
+                    let addr = {
+                        let base = callee_ret_area.unwrap().unwrap_sp();
+                        let slot_offset = base.as_u32() - *offset;
+                        self.masm.address_from_sp(SPOffset::from_u32(slot_offset))
+                    };
+
+                    self.masm.load(addr, self.alloc_scratch_reg, (*ty).into());
+                    self.masm.store(
+                        self.alloc_scratch_reg.into(),
+                        self.masm
+                            .address_at_reg(caller_retptr.unwrap(), *caller_offset),
+                        (*ty).into(),
+                    );
+                }
+                (ABIOperand::Stack { ty, offset, .. }, ABIOperand::Reg { reg, .. }) => {
+                    let addr = {
+                        let base = callee_ret_area.unwrap().unwrap_sp();
+                        let slot_offset = base.as_u32() - *offset;
+                        self.masm.address_from_sp(SPOffset::from_u32(slot_offset))
+                    };
+
+                    self.masm.load(addr, *reg, (*ty).into());
+                }
+                (ABIOperand::Reg { ty, .. }, ABIOperand::Reg { reg: dst, .. }) => {
+                    let spill_offset = spill_offsets_iter.next().unwrap();
+                    self.masm.load(
+                        self.masm.address_from_sp(*spill_offset),
+                        (*dst).into(),
+                        (*ty).into(),
+                    );
+                }
+            }
+        }
+        self.masm.free_stack(results_spill.1);
+    }
+
+    /// Emit a wasm-to-native trampoline.
+    pub fn emit_wasm_to_native(&mut self, ty: &WasmFuncType) -> Result<()> {
+        let mut params = self.callee_and_caller_vmctx_types();
+        params.extend_from_slice(ty.params());
+
+        let wasm_ty = WasmFuncType::new(params.into_boxed_slice(), ty.returns().into());
+        let wasm_sig = self.wasm_sig(&wasm_ty);
+        let native_sig = self.native_sig(ty);
+
+        let (vmctx, caller_vmctx) = Self::callee_and_caller_vmctx(&wasm_sig.params).unwrap();
+        let vmctx_runtime_limits_addr = self.vmctx_runtime_limits_addr(caller_vmctx);
+
+        self.prologue();
+
+        // Save the FP and return address when exiting Wasm.
+        // TODO: Once Winch supports comparison operators,
+        // check that the caller VM context is what we expect.
+        // See [`wasmtime_environ::MAGIC`].
+        Self::save_last_wasm_exit_fp_and_pc(
+            self.masm,
+            vmctx_runtime_limits_addr,
+            self.scratch_reg,
+            self.alloc_scratch_reg,
+            &self.pointer_size,
+        );
+
+        let ret_area = self.make_ret_area(&native_sig);
+        let (offsets, spill_size) = self.spill(wasm_sig.params());
+
+        let reserved_stack = self.masm.call(native_sig.params_stack_size(), |masm| {
+            // Move the VM context into one of the scratch registers.
+            masm.mov(
+                vmctx.into(),
+                self.alloc_scratch_reg.into(),
+                OperandSize::S64,
+            );
+
+            Self::assign_args(
+                masm,
+                &native_sig.params_without_retptr(),
+                &wasm_sig.params_without_retptr(),
+                &offsets,
+                self.scratch_reg,
+            );
+
+            Self::load_retptr(masm, ret_area.as_ref(), &native_sig);
+
+            let body_offset = self.pointer_size.vmnative_call_host_func_context_func_ref()
+                + self.pointer_size.vm_func_ref_native_call();
+            let callee_addr = masm.address_at_reg(self.alloc_scratch_reg, body_offset.into());
+            masm.load(callee_addr, self.scratch_reg, OperandSize::S64);
+
+            CalleeKind::Indirect(self.scratch_reg)
+        });
+
+        self.masm.free_stack(reserved_stack);
+        self.forward_results(&native_sig, &wasm_sig, ret_area.as_ref(), offsets.last());
+
+        if native_sig.results.has_stack_results() {
+            self.masm.free_stack(native_sig.results.size());
+        }
+
+        self.epilogue(spill_size);
+
+        Ok(())
+    }
+
+    /// Perfom argument assignment, translating between
+    /// caller and callee calling conventions.
+    fn assign_args(
+        masm: &mut M,
+        callee_params: &[ABIOperand],
+        caller_params: &[ABIOperand],
+        caller_stack_offsets: &[SPOffset],
+        scratch: Reg,
+    ) {
+        assert!(callee_params.len() == caller_params.len());
+        let arg_base_offset: u32 = <M::ABI as ABI>::arg_base_offset().into();
+        let fp = <M::ABI as ABI>::fp_reg();
+        let mut offset_index = 0;
+
+        callee_params
+            .iter()
+            .zip(caller_params)
+            .for_each(
+                |(callee_param, caller_param)| match (callee_param, caller_param) {
+                    (ABIOperand::Reg { ty, reg: dst, .. }, ABIOperand::Reg { .. }) => {
+                        let offset = caller_stack_offsets[offset_index];
+                        let addr = masm.address_from_sp(offset);
+                        masm.load(addr, *dst, (*ty).into());
+                        offset_index += 1;
+                    }
+
+                    (ABIOperand::Stack { ty, offset, .. }, ABIOperand::Reg { .. }) => {
+                        let spill_offset = caller_stack_offsets[offset_index];
+                        let addr = masm.address_from_sp(spill_offset);
+                        masm.load(addr, scratch, (*ty).into());
+
+                        let arg_addr = masm.address_at_sp(SPOffset::from_u32(*offset));
+                        masm.store(scratch.into(), arg_addr, (*ty).into());
+                        offset_index += 1;
+                    }
+
+                    (ABIOperand::Reg { ty, reg: dst, .. }, ABIOperand::Stack { offset, .. }) => {
+                        let addr = masm.address_at_reg(fp, arg_base_offset + offset);
+                        masm.load(addr, *dst, (*ty).into());
+                    }
+
+                    (
+                        ABIOperand::Stack {
+                            ty,
+                            offset: callee_offset,
+                            ..
+                        },
+                        ABIOperand::Stack {
+                            offset: caller_offset,
+                            ..
+                        },
+                    ) => {
+                        let addr = masm.address_at_reg(fp, arg_base_offset + caller_offset);
+                        masm.load(addr, scratch, (*ty).into());
+
+                        let arg_addr = masm.address_at_sp(SPOffset::from_u32(*callee_offset));
+                        masm.store(scratch.into(), arg_addr, (*ty).into());
+                    }
+                },
+            );
+    }
+
+    /// Get the type of the caller and callee VM contexts.
+    fn callee_and_caller_vmctx_types(&self) -> SmallVec<[WasmType; 2]> {
+        std::iter::repeat(self.pointer_type).take(2).collect()
+    }
+
+    /// Returns an [ABISig] for the array calling convention.
+    /// The signature looks like:
+    /// ```ignore
+    /// unsafe extern "C" fn(
+    ///     callee_vmctx: *mut VMOpaqueContext,
+    ///     caller_vmctx: *mut VMOpaqueContext,
+    ///     values_ptr: *mut ValRaw,
+    ///     values_len: usize,
+    /// )
+    /// ```
+    fn array_sig(&self) -> ABISig {
+        let mut params = self.callee_and_caller_vmctx_types();
+        params.extend_from_slice(&[self.pointer_type, self.pointer_type]);
+        <M::ABI as ABI>::sig_from(&params, &[], self.call_conv)
+    }
+
+    /// Returns an [ABISig] that follows a variation of the system's
+    /// calling convention.
+    /// The main difference between the flavor of the returned signature
+    /// and the vanilla signature is how multiple values are returned.
+    /// Multiple returns are handled following Wasmtime's expectations:
+    /// * A single value is returned via a register according to the calling
+    ///   convention.
+    /// * More than one values are returned via a return pointer.
+    /// These variations look like:
+    ///
+    /// Single return value.
+    ///
+    /// ```ignore
+    /// unsafe extern "C" fn(
+    ///     callee_vmctx: *mut VMOpaqueContext,
+    ///     caller_vmctx: *mut VMOpaqueContext,
+    ///     // rest of paramters
+    /// ) -> // single result
+    /// ```
+    ///
+    /// Multiple return values.
+    ///
+    /// ```ignore
+    /// unsafe extern "C" fn(
+    ///     callee_vmctx: *mut VMOpaqueContext,
+    ///     caller_vmctx: *mut VMOpaqueContext,
+    ///     // rest of parameters
+    ///     retptr: *mut (), // 2+ results
+    /// ) -> // first result
+    /// ```
+    fn native_sig(&self, ty: &WasmFuncType) -> ABISig {
+        let mut params = self.callee_and_caller_vmctx_types();
+        params.extend_from_slice(ty.params());
+        <M::ABI as ABI>::sig_from(&params, ty.returns(), self.call_conv)
+    }
+
+    /// Returns an [ABISig] using the Winch's default calling convention.
+    fn wasm_sig(&self, ty: &WasmFuncType) -> ABISig {
+        <M::ABI as ABI>::sig(ty, &CallingConvention::Default)
+    }
+
+    /// Returns the register pair containing the callee and caller VM context pointers.
+    fn callee_and_caller_vmctx(params: &ABIParams) -> Result<(Reg, Reg)> {
+        let vmctx = params
+            .get(0)
+            .map(|operand| operand.unwrap_reg())
+            .expect("Callee VMContext to be in a register");
+        let caller_vmctx = params
+            .get(1)
+            .map(|operand| operand.unwrap_reg())
+            .expect("Caller VMContext to be in a register");
+        Ok((vmctx, caller_vmctx))
+    }
+
+    /// Returns the address of the VM context runtime limits
+    /// field.
+    fn vmctx_runtime_limits_addr(&mut self, caller_vmctx: Reg) -> M::Address {
+        self.masm.address_at_reg(
+            caller_vmctx,
+            self.pointer_size.vmcontext_runtime_limits().into(),
+        )
+    }
+
+    /// Performs a spill of the given operands.
+    fn spill(&mut self, operands: &[ABIOperand]) -> (SmallVec<[SPOffset; 6]>, u32) {
+        let mut offsets = SmallVec::new();
+        let mut spill_size = 0;
+        operands.iter().for_each(|param| {
+            if let Some(reg) = param.get_reg() {
+                let slot = self.masm.push(reg, param.ty().into());
+                offsets.push(slot.offset);
+                spill_size += slot.size;
+            }
+        });
+
+        (offsets, spill_size)
+    }
+
+    /// Loads and assigns values from the value array used in the array
+    /// calling convention.
+    fn load_values_from_array(
+        masm: &mut M,
+        callee_sig: &ABISig,
+        ret_area: Option<&RetArea>,
+        values_reg: Reg,
+        scratch: Reg,
+    ) {
+        callee_sig
+            .params_without_retptr()
+            .iter()
+            .enumerate()
+            .for_each(|(i, param)| {
+                let value_offset = (i * VALUE_SIZE) as u32;
+
+                match param {
+                    ABIOperand::Reg { reg, ty, .. } => masm.load(
+                        masm.address_at_reg(values_reg, value_offset),
+                        *reg,
+                        (*ty).into(),
+                    ),
+                    ABIOperand::Stack { offset, ty, .. } => {
+                        masm.load(
+                            masm.address_at_reg(values_reg, value_offset),
+                            scratch,
+                            (*ty).into(),
+                        );
+                        masm.store(
+                            scratch.into(),
+                            masm.address_at_sp(SPOffset::from_u32(*offset)),
+                            (*ty).into(),
+                        );
+                    }
+                }
+            });
+
+        // Assign the retpr param.
+        if let Some(offs) = ret_area {
+            let results_area_operand = callee_sig.params.unwrap_results_area_operand();
+            let addr = match offs {
+                RetArea::SP(sp_offset) => masm.address_from_sp(*sp_offset),
+                _ => unreachable!(),
+            };
+            match results_area_operand {
+                ABIOperand::Reg { ty, reg, .. } => {
+                    masm.load_addr(addr, (*reg).into(), (*ty).into());
+                }
+                ABIOperand::Stack { ty, offset, .. } => {
+                    masm.load_addr(addr, scratch, (*ty).into());
+                    masm.store(
+                        scratch.into(),
+                        masm.address_at_sp(SPOffset::from_u32(*offset)),
+                        (*ty).into(),
+                    );
+                }
+            }
+        }
+    }
+
+    fn save_last_wasm_entry_sp(
+        masm: &mut M,
+        vm_runtime_limits_addr: M::Address,
+        scratch: Reg,
+        ptr: &impl PtrSize,
+    ) {
+        let sp = <M::ABI as ABI>::sp_reg();
+        masm.load(vm_runtime_limits_addr, scratch, OperandSize::S64);
+        let addr = masm.address_at_reg(scratch, ptr.vmruntime_limits_last_wasm_entry_sp().into());
+        masm.store(sp.into(), addr, OperandSize::S64);
+    }
+
+    fn save_last_wasm_exit_fp_and_pc(
+        masm: &mut M,
+        vm_runtime_limits_addr: M::Address,
+        scratch: Reg,
+        alloc_scratch: Reg,
+        ptr: &impl PtrSize,
+    ) {
+        masm.load(vm_runtime_limits_addr, alloc_scratch, OperandSize::S64);
+        let last_wasm_exit_fp_addr = masm.address_at_reg(
+            alloc_scratch,
+            ptr.vmruntime_limits_last_wasm_exit_fp().into(),
+        );
+        let last_wasm_exit_pc_addr = masm.address_at_reg(
+            alloc_scratch,
+            ptr.vmruntime_limits_last_wasm_exit_pc().into(),
+        );
+
+        // Handle the frame pointer.
+        let fp = <M::ABI as ABI>::fp_reg();
+        let fp_addr = masm.address_at_reg(fp, 0);
+        masm.load(fp_addr, scratch, OperandSize::S64);
+        masm.store(scratch.into(), last_wasm_exit_fp_addr, OperandSize::S64);
+
+        // Handle the return address.
+        let ret_addr_offset = <M::ABI as ABI>::ret_addr_offset();
+        let ret_addr = masm.address_at_reg(fp, ret_addr_offset.into());
+        masm.load(ret_addr, scratch, OperandSize::S64);
+        masm.store(scratch.into(), last_wasm_exit_pc_addr, OperandSize::S64);
+    }
+
+    /// The trampoline's prologue.
+    fn prologue(&mut self) {
+        self.masm.prologue();
+    }
+
+    /// Similar to [Trampoline::prologue], but saves
+    /// callee-saved registers.
+    fn prologue_with_callee_saved(&mut self) {
+        self.masm.prologue();
+        // Save any callee-saved registers.
+        for (r, s) in &self.callee_saved_regs {
+            self.masm.push(*r, *s);
+        }
+    }
+
+    /// Similar to [Trampoline::epilogue], but restores
+    /// callee-saved registers.
+    fn epilogue_with_callee_saved_restore(&mut self, arg_size: u32) {
+        // Free the stack space allocated by pushing the trampoline arguments.
+        self.masm.free_stack(arg_size);
+        // Restore the callee-saved registers.
+        for (r, s) in self.callee_saved_regs.iter().rev() {
+            self.masm.pop(*r, *s);
+        }
+        self.masm.epilogue(0);
+    }
+
+    /// The trampoline's epilogue.
+    fn epilogue(&mut self, arg_size: u32) {
+        // Free the stack space allocated by pushing the trampoline arguments.
+        self.masm.free_stack(arg_size);
+        self.masm.epilogue(0);
+    }
+}