wasmtime 21.0.1 → 22.0.0

data/ext/cargo-vendor/winch-codegen-0.19.1/src/codegen/control.rs

@@ -1,972 +0,0 @@
-//! Data structures for control flow emission.
-//!
-//! Winch currently doesn't apply any sort of optimizations to control flow, but
-//! as a future optimization, for starters, we could perform a look ahead to the
-//! next instruction when reaching any of the comparison instructions. If the
-//! next instruction is a control instruction, we could avoid emitting
-//! a [`crate::masm::MacroAssembler::cmp_with_set`] and instead emit
-//! a conditional jump inline when emitting the control flow instruction.
-use super::{CodeGenContext, OperandSize, Reg, TypedReg};
-use crate::{
-    abi::{ABIOperand, ABIResults, ABISig, RetArea, ABI},
-    masm::{IntCmpKind, MacroAssembler, MemMoveDirection, RegImm, SPOffset},
-    stack::Val,
-    CallingConvention,
-};
-use cranelift_codegen::MachLabel;
-use wasmtime_environ::{WasmFuncType, WasmValType};
-
-/// Categorization of the type of the block.
-#[derive(Debug, Clone)]
-pub(crate) enum BlockType {
-    /// Doesn't produce or consume any values.
-    Void,
-    /// Produces a single value.
-    Single(WasmValType),
-    /// Consumes multiple values and produces multiple values.
-    Func(WasmFuncType),
-    /// An already resolved ABI signature.
-    ABISig(ABISig),
-}
-
-/// Holds all the information about the signature of the block.
-#[derive(Debug, Clone)]
-pub(crate) struct BlockSig {
-    /// The type of the block.
-    pub ty: BlockType,
-    /// ABI representation of the results of the block.
-    results: Option<ABIResults>,
-    /// ABI representation of the params of the block interpreted as results.
-    params: Option<ABIResults>,
-}
-
-impl BlockSig {
-    /// Create a new [BlockSig].
-    pub fn new(ty: BlockType) -> Self {
-        Self {
-            ty,
-            results: None,
-            params: None,
-        }
-    }
-
-    /// Create a new [BlockSig] from an [ABISig].
-    pub fn from_sig(sig: ABISig) -> Self {
-        Self {
-            ty: BlockType::sig(sig),
-            results: None,
-            params: None,
-        }
-    }
-
-    /// Return the ABI representation of the results of the block.
-    /// This method will lazily initialize the results if not present.
-    pub fn results<M>(&mut self) -> &mut ABIResults
-    where
-        M: MacroAssembler,
-    {
-        if self.ty.is_sig() {
-            return match &mut self.ty {
-                BlockType::ABISig(sig) => &mut sig.results,
-                _ => unreachable!(),
-            };
-        }
-
-        if self.results.is_some() {
-            return self.results.as_mut().unwrap();
-        }
-
-        let results = match &self.ty {
-            BlockType::Void => <M::ABI as ABI>::abi_results(&[], &CallingConvention::Default),
-            BlockType::Single(ty) => {
-                <M::ABI as ABI>::abi_results(&[*ty], &CallingConvention::Default)
-            }
-            BlockType::Func(f) => {
-                <M::ABI as ABI>::abi_results(f.returns(), &CallingConvention::Default)
-            }
-            BlockType::ABISig(_) => unreachable!(),
-        };
-
-        self.results = Some(results);
-        self.results.as_mut().unwrap()
-    }
-
-    /// Construct an ABI result representation of the params of the block.
-    /// This is needed for loops and for handling cases in which params flow as
-    /// the block's results, i.e. in the presence of an empty then or else.
-    pub fn params<M>(&mut self) -> &mut ABIResults
-    where
-        M: MacroAssembler,
-    {
-        if self.params.is_some() {
-            return self.params.as_mut().unwrap();
-        }
-
-        let params_as_results = match &self.ty {
-            BlockType::Void | BlockType::Single(_) => {
-                <M::ABI as ABI>::abi_results(&[], &CallingConvention::Default)
-            }
-            BlockType::Func(f) => {
-                <M::ABI as ABI>::abi_results(f.params(), &CallingConvention::Default)
-            }
-            // Once we have created a block type from a known signature, we
-            // can't modify its meaning. This should only be used for the
-            // function body block, in which case there's no need for treating
-            // params as results.
-            BlockType::ABISig(_) => unreachable!(),
-        };
-
-        self.params = Some(params_as_results);
-        self.params.as_mut().unwrap()
-    }
-
-    /// Returns the signature param count.
-    pub fn param_count(&self) -> usize {
-        match &self.ty {
-            BlockType::Void | BlockType::Single(_) => 0,
-            BlockType::Func(f) => f.params().len(),
-            BlockType::ABISig(sig) => sig.params_without_retptr().len(),
-        }
-    }
-
-    /// Returns the signature return count.
-    pub fn return_count(&self) -> usize {
-        match &self.ty {
-            BlockType::Void => 0,
-            BlockType::Single(_) => 1,
-            BlockType::Func(f) => f.returns().len(),
-            BlockType::ABISig(sig) => sig.results().len(),
-        }
-    }
-}
-
-impl BlockType {
-    /// Create a [BlockType::Void].
-    pub fn void() -> Self {
-        Self::Void
-    }
-
-    /// Create a [BlockType::Single] from the given [WasmType].
-    pub fn single(ty: WasmValType) -> Self {
-        Self::Single(ty)
-    }
-
-    /// Create a [BlockType::Func] from the given [WasmFuncType].
-    pub fn func(ty: WasmFuncType) -> Self {
-        Self::Func(ty)
-    }
-
-    /// Create a [BlockType::ABISig].
-    pub fn sig(sig: ABISig) -> Self {
-        Self::ABISig(sig)
-    }
-
-    /// Returns true if the type of the block is [BlockType::ABISig].
-    pub fn is_sig(&self) -> bool {
-        match self {
-            Self::ABISig(_) => true,
-            _ => false,
-        }
-    }
-}
-
-/// The expected value and machine stack state when entering and exiting the block.
-#[derive(Debug, Default, Copy, Clone)]
-pub(crate) struct StackState {
-    /// The base stack pointer offset.
-    /// This offset is set when entering the block, after saving any live
-    /// registers and locals.
-    /// It is calcuated by substracting the size, in bytes, of any block params
-    /// to the current stack pointer offset.
-    pub base_offset: SPOffset,
-    /// The target stack pointer offset.
-    /// This offset is calculated by adding the size of the stack results
-    /// to the base stack pointer offset.
-    pub target_offset: SPOffset,
-    /// The base length of the value stack when entering the block.
-    /// Which is the current length of the value stack minus any block parameters.
-    pub base_len: usize,
-    /// The target length of the value stack when exiting the block.
-    /// Calculate by adding the number of results to the base value stack
-    /// length.
-    pub target_len: usize,
-}
-
-/// Holds the all the metdata to support the emission
-/// of control flow instructions.
-#[derive(Debug)]
-pub(crate) enum ControlStackFrame {
-    If {
-        /// The if continuation label.
-        cont: MachLabel,
-        /// The exit label of the block.
-        exit: MachLabel,
-        /// The signature of the block.
-        sig: BlockSig,
-        /// The stack state of the block.
-        stack_state: StackState,
-        /// Local reachability state when entering the block.
-        reachable: bool,
-    },
-    Else {
-        /// The exit label of the block.
-        exit: MachLabel,
-        /// The signature of the block.
-        sig: BlockSig,
-        /// The stack state of the block.
-        stack_state: StackState,
-        /// Local reachability state when entering the block.
-        reachable: bool,
-    },
-    Block {
-        /// The block exit label.
-        exit: MachLabel,
-        /// The signature of the block.
-        sig: BlockSig,
-        /// The stack state of the block.
-        stack_state: StackState,
-        /// Exit state of the block.
-        ///
-        /// This flag is used to dertermine if a block is a branch
-        /// target. By default, this is false, and it's updated when
-        /// emitting a `br` or `br_if`.
-        is_branch_target: bool,
-    },
-    Loop {
-        /// The start of the Loop.
-        head: MachLabel,
-        /// The stack state of the block.
-        stack_state: StackState,
-        /// The signature of the block.
-        sig: BlockSig,
-    },
-}
-
-impl ControlStackFrame {
-    /// Returns [`ControlStackFrame`] for an if.
-    pub fn r#if<M: MacroAssembler>(
-        sig: BlockSig,
-        masm: &mut M,
-        context: &mut CodeGenContext,
-    ) -> Self {
-        let mut control = Self::If {
-            cont: masm.get_label(),
-            exit: masm.get_label(),
-            sig,
-            reachable: context.reachable,
-            stack_state: Default::default(),
-        };
-
-        control.emit(masm, context);
-        control
-    }
-
-    /// Returns [`ControlStackFrame`] for a block.
-    pub fn block<M: MacroAssembler>(
-        sig: BlockSig,
-        masm: &mut M,
-        context: &mut CodeGenContext,
-    ) -> Self {
-        let mut control = Self::Block {
-            sig,
-            is_branch_target: false,
-            exit: masm.get_label(),
-            stack_state: Default::default(),
-        };
-
-        control.emit(masm, context);
-        control
-    }
-
-    /// Returns [`ControlStackFrame`] for a loop.
-    pub fn r#loop<M: MacroAssembler>(
-        sig: BlockSig,
-        masm: &mut M,
-        context: &mut CodeGenContext,
-    ) -> Self {
-        let mut control = Self::Loop {
-            stack_state: Default::default(),
-            sig,
-            head: masm.get_label(),
-        };
-
-        control.emit(masm, context);
-        control
-    }
-
-    fn init<M: MacroAssembler>(&mut self, masm: &mut M, context: &mut CodeGenContext) {
-        self.calculate_stack_state(context, masm);
-        // If the block has stack results, immediately resolve the return area
-        // base.
-        if self.results::<M>().on_stack() {
-            let results_base = self.stack_state().target_offset;
-            self.results::<M>().set_ret_area(RetArea::sp(results_base));
-        }
-
-        if self.is_if() || self.is_loop() {
-            // Preemptively handle block params as results so that the params
-            // are correctly placed in memory. This is especially
-            // important for control flow joins with empty blocks:
-            //
-            //(module
-            //  (func (export "params") (param i32) (result i32)
-            //       (i32.const 2)
-            //       (if (param i32) (result i32) (local.get 0)
-            //       (then))
-            //     (i32.const 3)
-            //     (i32.add)
-            //   )
-            //)
-            let base_offset = self.stack_state().base_offset;
-            if self.params::<M>().on_stack() {
-                let offset = base_offset.as_u32() + self.params::<M>().size();
-                self.params::<M>()
-                    .set_ret_area(RetArea::sp(SPOffset::from_u32(offset)));
-            }
-            Self::top_abi_results_impl(
-                self.params::<M>(),
-                context,
-                masm,
-                |params: &ABIResults, _, _| params.ret_area().copied(),
-            );
-        }
-    }
-
-    /// Calculates the [StackState] of the block.
-    fn calculate_stack_state<M: MacroAssembler>(
-        &mut self,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-    ) {
-        use ControlStackFrame::*;
-        let sig = self.sig();
-        // If the block type contains a full [ABISig], do not take into account
-        // the params, since these are the params of the function that is
-        // currently being compiled and the value stack doesn't currently
-        // contain any values anyway.
-        let param_count = if sig.ty.is_sig() {
-            0
-        } else {
-            sig.param_count()
-        };
-        let return_count = sig.return_count();
-        debug_assert!(context.stack.len() >= param_count);
-        let results_size = self.results::<M>().size();
-
-        // Save any live registers and locals.
-        context.spill(masm);
-
-        let base_len = context.stack.len() - param_count;
-        let stack_consumed = context.stack.sizeof(param_count);
-        let current_sp = masm.sp_offset();
-        let base_offset = SPOffset::from_u32(current_sp.as_u32() - stack_consumed);
-
-        match self {
-            If { stack_state, .. } | Block { stack_state, .. } | Loop { stack_state, .. } => {
-                stack_state.base_offset = base_offset;
-                stack_state.base_len = base_len;
-                stack_state.target_offset = SPOffset::from_u32(base_offset.as_u32() + results_size);
-                stack_state.target_len = base_len + return_count;
-            }
-            _ => {}
-        }
-    }
-
-    /// This function ensures that the state of the -- machine and value --
-    /// stack  is the right one when reaching a control frame branch in which
-    /// reachability is restored or when reaching the end of a function in an
-    /// unreachable state. This function is intended to be called when handling
-    /// an unreachable else or end.
-    //
-    /// This function will truncate the value stack to the the base length of
-    /// the control frame and will also set the stack pointer offset to reflect
-    /// the offset expected by the target branch.
-    ///
-    // NB: This method is assumed to be called *before* pushing any block
-    // results to the value stack, so that any excess values are cleaned up.
-    pub fn ensure_stack_state<M: MacroAssembler>(
-        &mut self,
-        masm: &mut M,
-        context: &mut CodeGenContext,
-    ) {
-        let state = self.stack_state();
-        // This assumes that at jump sites, the machine stack poiter will be
-        // adjusted to match the expectations of the target branch (e.g.
-        // `target_offset`); after performing the jump, the MacroAssembler
-        // implementation will soft-reset the stack pointer offset to its
-        // original offset, ensure that other parts of the program have access
-        // to the right offset, this is especially important in conditional
-        // branches.
-        // When restoring reachability we ensure that the MacroAssembler offset
-        // is set to match the expectations of the target branch, similar to how
-        // the machine stack pointer was adjusted at jump sites.
-        masm.reset_stack_pointer(state.target_offset);
-        // We use the base length, because this function is assumed to be called
-        // *before* pushing any results to the value stack. This way, any excess
-        // values will be discarded.
-        context.truncate_stack_to(state.base_len);
-    }
-
-    /// Return the type information of the block.
-    pub fn sig(&self) -> &BlockSig {
-        use ControlStackFrame::*;
-        match self {
-            If { sig, .. } | Else { sig, .. } | Loop { sig, .. } | Block { sig, .. } => sig,
-        }
-    }
-
-    fn emit<M: MacroAssembler>(&mut self, masm: &mut M, context: &mut CodeGenContext) {
-        use ControlStackFrame::*;
-
-        // Do not perform any emissions if we are in an unreachable state.
-        if !context.reachable {
-            return;
-        }
-
-        match *self {
-            If { cont, .. } => {
-                // Pop the condition value.
-                // Because in the case of Self::If, Self::init, will top the
-                // branch params, we exclude any result registers from being
-                // used as the branch test.
-                let top =
-                    context.without::<_, _, _>(self.params::<M>().regs(), masm, |cx, masm| {
-                        cx.pop_to_reg(masm, None)
-                    });
-                self.init(masm, context);
-                masm.branch(
-                    IntCmpKind::Eq,
-                    top.reg.into(),
-                    top.reg.into(),
-                    cont,
-                    OperandSize::S32,
-                );
-                context.free_reg(top);
-            }
-            Block { .. } => self.init(masm, context),
-            Loop { head, .. } => {
-                self.init(masm, context);
-                masm.bind(head);
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    /// Handles the else branch if the current control stack frame is
-    /// [`ControlStackFrame::If`].
-    pub fn emit_else<M: MacroAssembler>(&mut self, masm: &mut M, context: &mut CodeGenContext) {
-        debug_assert!(self.is_if());
-        let state = self.stack_state();
-
-        debug_assert_eq!(state.target_len, context.stack.len());
-        self.pop_abi_results(context, masm, |results, _, _| results.ret_area().copied());
-        masm.jmp(*self.exit_label().unwrap());
-        self.bind_else(masm, context);
-    }
-
-    /// Binds the else branch label and converts `self` to
-    /// [`ControlStackFrame::Else`].
-    pub fn bind_else<M: MacroAssembler>(&mut self, masm: &mut M, context: &mut CodeGenContext) {
-        use ControlStackFrame::*;
-        match self {
-            If {
-                cont,
-                sig,
-                stack_state,
-                exit,
-                ..
-            } => {
-                // Bind the else branch.
-                masm.bind(*cont);
-
-                // Push the abi results to the value stack, so that they are
-                // used as params for the else branch. At the beginning of the
-                // if block, any params are preemptively resolved as results;
-                // when reaching the else all params are already materialized as
-                // stack results. As part of ensuring the right state when
-                // entering the else branch, the following snippet also soft
-                // resets the stack pointer so that it matches the expectations
-                // of the else branch: the stack pointer is expected to be at
-                // the base stack pointer, plus the params stack size in bytes.
-                let params_size = sig.params::<M>().size();
-                context.push_abi_results::<M, _>(sig.params::<M>(), masm, |params, _, _| {
-                    params.ret_area().copied()
-                });
-                masm.reset_stack_pointer(SPOffset::from_u32(
-                    stack_state.base_offset.as_u32() + params_size,
-                ));
-
-                // Update the stack control frame with an else control frame.
-                *self = ControlStackFrame::Else {
-                    exit: *exit,
-                    stack_state: *stack_state,
-                    reachable: context.reachable,
-                    sig: sig.clone(),
-                };
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    /// Handles the end of a control stack frame.
-    pub fn emit_end<M: MacroAssembler>(&mut self, masm: &mut M, context: &mut CodeGenContext) {
-        use ControlStackFrame::*;
-        match self {
-            If { stack_state, .. } | Else { stack_state, .. } | Block { stack_state, .. } => {
-                debug_assert_eq!(stack_state.target_len, context.stack.len());
-                // Before binding the exit label, we handle the block results.
-                self.pop_abi_results(context, masm, |results, _, _| results.ret_area().copied());
-                self.bind_end(masm, context);
-            }
-            Loop { stack_state, .. } => {
-                debug_assert_eq!(stack_state.target_len, context.stack.len());
-            }
-        }
-    }
-
-    /// Binds the exit label of the current control stack frame and pushes the
-    /// ABI results to the value stack.
-    pub fn bind_end<M: MacroAssembler>(&mut self, masm: &mut M, context: &mut CodeGenContext) {
-        self.push_abi_results(context, masm);
-        self.bind_exit_label(masm);
-    }
-
-    /// Binds the exit label of the control stack frame.
-    pub fn bind_exit_label<M: MacroAssembler>(&self, masm: &mut M) {
-        use ControlStackFrame::*;
-        match self {
-            // We use an explicit label to track the exit of an if block. In case there's no
-            // else, we bind the if's continuation block to make sure that any jumps from the if
-            // condition are reachable and we bind the explicit exit label as well to ensure that any
-            // branching instructions are able to correctly reach the block's end.
-            If { cont, .. } => masm.bind(*cont),
-            _ => {}
-        }
-        if let Some(label) = self.exit_label() {
-            masm.bind(*label);
-        }
-    }
-
-    /// Returns the continuation label of the current control stack frame.
-    pub fn label(&self) -> &MachLabel {
-        use ControlStackFrame::*;
-
-        match self {
-            If { exit, .. } | Else { exit, .. } | Block { exit, .. } => exit,
-            Loop { head, .. } => head,
-        }
-    }
-
-    /// Returns the exit label of the current control stack frame. Note that
-    /// this is similar to [`ControlStackFrame::label`], with the only difference that it
-    /// returns `None` for `Loop` since its label doesn't represent an exit.
-    pub fn exit_label(&self) -> Option<&MachLabel> {
-        use ControlStackFrame::*;
-
-        match self {
-            If { exit, .. } | Else { exit, .. } | Block { exit, .. } => Some(exit),
-            Loop { .. } => None,
-        }
-    }
-
-    /// Set the current control stack frame as a branch target.
-    pub fn set_as_target(&mut self) {
-        match self {
-            ControlStackFrame::Block {
-                is_branch_target, ..
-            } => {
-                *is_branch_target = true;
-            }
-            _ => {}
-        }
-    }
-
-    /// Returns [`crate::abi::ABIResults`] of the control stack frame
-    /// block.
-    pub fn results<M>(&mut self) -> &mut ABIResults
-    where
-        M: MacroAssembler,
-    {
-        use ControlStackFrame::*;
-
-        match self {
-            If { sig, .. } | Else { sig, .. } | Block { sig, .. } => sig.results::<M>(),
-            Loop { sig, .. } => sig.params::<M>(),
-        }
-    }
-
-    /// Returns the block params interpreted as [crate::abi::ABIResults].
-    pub fn params<M>(&mut self) -> &mut ABIResults
-    where
-        M: MacroAssembler,
-    {
-        use ControlStackFrame::*;
-        match self {
-            If { sig, .. } | Else { sig, .. } | Block { sig, .. } | Loop { sig, .. } => {
-                sig.params::<M>()
-            }
-        }
-    }
-
-    /// Orchestrates how blocks results are handled.
-    /// Results are handled in reverse order, starting from register results
-    /// continuing to memory values. This guarantees that the stack ordering
-    /// invariant is maintained. See [ABIResults] for more details.
-    ///
-    /// This function will iterate through each result and invoke the provided
-    /// callback if there are results on the stack.
-    ///
-    /// Calculating the return area involves ensuring that there's enough stack
-    /// space to store the block's results. To make the process of handling
-    /// multiple results easier, this function will save all live registers and
-    /// locals right after handling any register results. This will ensure that
-    /// the top `n` values in the value stack are correctly placed in the memory
-    /// locations corresponding to multiple stack results. Once the iteration
-    /// over all the results is done, the stack result area of the block will be
-    /// updated.
-    pub fn pop_abi_results<M, F>(
-        &mut self,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-        calculate_ret_area: F,
-    ) where
-        M: MacroAssembler,
-        F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option<RetArea>,
-    {
-        Self::pop_abi_results_impl(self.results::<M>(), context, masm, calculate_ret_area)
-    }
-
-    /// Shared implementation for poppping the ABI results.
-    /// This is needed because, in some cases, params must be interpreted and
-    /// used as the results of the block. When emitting code at control flow
-    /// joins, the block params are interpreted as results, to ensure that they
-    /// can correctly "flow" as the results of the block. This is especially
-    /// important in the presence of empty then, else and loop blocks. This
-    /// interpretation is an internal detail of the control module, and having
-    /// a shared implementation allows the caller to decide how the
-    /// results should be interpreted.
-    pub fn pop_abi_results_impl<M, F>(
-        results: &mut ABIResults,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-        mut calculate_ret_area: F,
-    ) where
-        M: MacroAssembler,
-        F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option<RetArea>,
-    {
-        let mut iter = results.operands().iter().rev().peekable();
-
-        while let Some(ABIOperand::Reg { reg, .. }) = iter.peek() {
-            let TypedReg { reg, .. } = context.pop_to_reg(masm, Some(*reg));
-            context.free_reg(reg);
-            iter.next().unwrap();
-        }
-
-        let ret_area = calculate_ret_area(results, context, masm);
-
-        let retptr = Self::maybe_load_retptr(ret_area.as_ref(), &results, context, masm);
-        if let Some(area) = ret_area {
-            if area.is_sp() {
-                Self::ensure_ret_area(&area, context, masm);
-            }
-        }
-
-        if let Some(retptr) = retptr {
-            while let Some(ABIOperand::Stack { offset, .. }) = iter.peek() {
-                let addr = masm.address_at_reg(retptr, *offset);
-                context.pop_to_addr(masm, addr);
-                iter.next().unwrap();
-            }
-            context.free_reg(retptr);
-        }
-
-        if let Some(area) = ret_area {
-            if area.is_sp() {
-                Self::adjust_stack_results(area, results, context, masm);
-            }
-        }
-    }
-
-    /// Convenience wrapper around [CodeGenContext::push_abi_results] using the
-    /// results of the current frame.
-    fn push_abi_results<M>(&mut self, context: &mut CodeGenContext, masm: &mut M)
-    where
-        M: MacroAssembler,
-    {
-        context.push_abi_results(self.results::<M>(), masm, |results, _, _| {
-            results.ret_area().copied()
-        })
-    }
-
-    /// Preemptively handles the ABI results of the current frame.
-    /// This function is meant to be used when emitting control flow with joins,
-    /// in which it's not possible to know at compile time which branch will be
-    /// taken.
-    pub fn top_abi_results<M, F>(
-        &mut self,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-        calculate_ret_area: F,
-    ) where
-        M: MacroAssembler,
-        F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option<RetArea>,
-    {
-        Self::top_abi_results_impl::<M, _>(self.results::<M>(), context, masm, calculate_ret_area)
-    }
-
-    /// Internal implementation of [Self::top_abi_results].
-    /// See [Self::pop_abi_results_impl] on why an internal implementation is
-    /// needed.
-    fn top_abi_results_impl<M, F>(
-        results: &mut ABIResults,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-        mut calculate_ret_area: F,
-    ) where
-        M: MacroAssembler,
-        F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option<RetArea>,
-    {
-        let mut area = None;
-        Self::pop_abi_results_impl::<M, _>(results, context, masm, |r, context, masm| {
-            area = calculate_ret_area(r, context, masm);
-            area
-        });
-        // Use the previously calculated area to ensure that the ret area is
-        // kept in sync between both operations.
-        context.push_abi_results::<M, _>(results, masm, |_, _, _| area);
-    }
-
-    // If the results on the stack are handled via the stack pointer, ensure
-    // that the stack results are correctly located. In general, since values in
-    // the value stack are spilled when exiting the block, the top `n` entries
-    // in the value stack, representing the `n` stack results of the block are
-    // almost correctly located. However, since constants are not
-    // spilled, their presence complicate block exits. For this reason, the
-    // last step for finalizing multiple block results involves:
-    // * Scanning the value stack from oldest to newest memory values and
-    // calculating the source and destination of each value, if the source
-    // is closer to the stack pointer (greater) than the destination,
-    // perform a memory move of the bytes to its destination, else stop,
-    // because the memory values are in place.
-    // * Scanning the value stack from newest to oldest and calculating the
-    // source and destination of each value, if the source is closer to the
-    // frame pointer (less) than the destination, perform a memory move of
-    // the bytes to its destination, else stop, because the memory values
-    // are in place.
-    // * Lastly, iterate over the top `n` elements of the value stack,
-    // and spill any constant values, placing them in their respective
-    // memory location.
-    //
-    // The implementation in Winch is inspired by how this is handled in
-    // SpiderMonkey's WebAssembly Baseline Compiler:
-    // https://wingolog.org/archives/2020/04/03/multi-value-webassembly-in-firefox-from-1-to-n
-    fn adjust_stack_results<M>(
-        ret_area: RetArea,
-        results: &ABIResults,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-    ) where
-        M: MacroAssembler,
-    {
-        debug_assert!(ret_area.is_sp());
-        let results_offset = ret_area.unwrap_sp();
-
-        // Start iterating from memory values that are closer to the
-        // frame pointer (oldest entries first).
-        for (i, operand) in results.operands().iter().enumerate() {
-            if operand.is_reg() {
-                break;
-            }
-
-            let value_index = (context.stack.len() - results.stack_operands_len()) + i;
-            let val = context.stack.inner()[value_index];
-
-            match (val, operand) {
-                (Val::Memory(mem), ABIOperand::Stack { offset, size, .. }) => {
-                    let dst = results_offset.as_u32() - *offset;
-                    let src = mem.slot.offset;
-
-                    // Values are moved from lower (SP) to higher (FP)
-                    // addresses.
-                    if src.as_u32() <= dst {
-                        break;
-                    }
-
-                    masm.memmove(
-                        src,
-                        SPOffset::from_u32(dst),
-                        *size,
-                        MemMoveDirection::LowToHigh,
-                    );
-                }
-                _ => {}
-            }
-        }
-
-        // Start iterating from memory values that are closer to the
-        // stack pointer (newest entries first).
-        for (i, operand) in results
-            .operands()
-            .iter()
-            .rev()
-            // Skip any register results.
-            .skip(results.regs().len())
-            .enumerate()
-        {
-            let value_index = context.stack.len() - i - 1;
-            let val = context.stack.inner()[value_index];
-            match (val, operand) {
-                (Val::Memory(mem), ABIOperand::Stack { offset, size, .. }) => {
-                    let dst = results_offset.as_u32() - *offset;
-                    let src = mem.slot.offset;
-
-                    // Values are moved from higher (FP) to lower (SP)
-                    // addresses.
-                    if src.as_u32() >= dst {
-                        break;
-                    }
-
-                    masm.memmove(
-                        src,
-                        SPOffset::from_u32(dst),
-                        *size,
-                        MemMoveDirection::HighToLow,
-                    );
-                }
-                _ => {}
-            }
-        }
-
-        // Finally store any constants in the value stack in their respective
-        // locations.
-        for operand in results
-            .operands()
-            .iter()
-            .take(results.stack_operands_len())
-            .rev()
-        {
-            // If we want to do this, we should start from newest, essentially from top to
-            // bottom in the iteration of the operands.
-            match (operand, context.stack.peek().unwrap()) {
-                (ABIOperand::Stack { ty, offset, .. }, Val::I32(v)) => {
-                    let addr =
-                        masm.address_from_sp(SPOffset::from_u32(results_offset.as_u32() - *offset));
-                    masm.store(RegImm::i32(*v), addr, (*ty).into());
-                }
-                (ABIOperand::Stack { ty, offset, .. }, Val::I64(v)) => {
-                    let addr =
-                        masm.address_from_sp(SPOffset::from_u32(results_offset.as_u32() - *offset));
-                    masm.store(RegImm::i64(*v), addr, (*ty).into());
-                }
-                (ABIOperand::Stack { ty, offset, .. }, Val::F32(v)) => {
-                    let addr =
-                        masm.address_from_sp(SPOffset::from_u32(results_offset.as_u32() - *offset));
-                    masm.store(RegImm::f32(v.bits()), addr, (*ty).into());
-                }
-                (ABIOperand::Stack { ty, offset, .. }, Val::F64(v)) => {
-                    let addr =
-                        masm.address_from_sp(SPOffset::from_u32(results_offset.as_u32() - *offset));
-                    masm.store(RegImm::f64(v.bits()), addr, (*ty).into());
-                }
-                (_, v) => debug_assert!(v.is_mem()),
-            }
-
-            let _ = context.stack.pop().unwrap();
-        }
-
-        // Adjust any excess stack space: the stack space after handling the
-        // block's results should be the exact amount needed by the return area.
-        debug_assert!(masm.sp_offset().as_u32() >= results_offset.as_u32());
-        masm.free_stack(masm.sp_offset().as_u32() - results_offset.as_u32());
-    }
-
-    /// Ensures that there is enough space for return values on the stack.
-    /// This function is called at the end of all blocks and when branching from
-    /// within blocks.
-    fn ensure_ret_area<M>(ret_area: &RetArea, context: &mut CodeGenContext, masm: &mut M)
-    where
-        M: MacroAssembler,
-    {
-        debug_assert!(ret_area.is_sp());
-        // Save any live registers and locals when exiting the block to ensure
-        // that the respective values are correctly located in memory.
-        // See [Self::adjust_stack_results] for more details.
-        context.spill(masm);
-        if ret_area.unwrap_sp() > masm.sp_offset() {
-            masm.reserve_stack(ret_area.unwrap_sp().as_u32() - masm.sp_offset().as_u32())
-        }
-    }
-
-    /// Loads the return pointer, if it exists, into the next available register.
-    fn maybe_load_retptr<M>(
-        ret_area: Option<&RetArea>,
-        results: &ABIResults,
-        context: &mut CodeGenContext,
-        masm: &mut M,
-    ) -> Option<Reg>
-    where
-        M: MacroAssembler,
-    {
-        ret_area
-            .map(|area| match area {
-                RetArea::Slot(slot) => {
-                    let base = context
-                        .without::<_, M, _>(results.regs(), masm, |cx, masm| cx.any_gpr(masm));
-                    let local_addr = masm.local_address(&slot);
-                    masm.load_ptr(local_addr, base);
-                    Some(base)
-                }
-                _ => None,
-            })
-            .flatten()
-    }
-
-    /// This function is used at the end of unreachable code handling
-    /// to determine if the reachability status should be updated.
-    pub fn is_next_sequence_reachable(&self) -> bool {
-        use ControlStackFrame::*;
-
-        match self {
-            // For if/else, the reachability of the next sequence is determined
-            // by the reachability state at the start of the block. An else
-            // block will be reachable if the if block is also reachable at
-            // entry.
-            If { reachable, .. } | Else { reachable, .. } => *reachable,
-            // For blocks, the reachability of the next sequence is determined
-            // if they're a branch target.
-            Block {
-                is_branch_target, ..
-            } => *is_branch_target,
-            // Loops are not used for reachability analysis,
-            // given that they don't have exit branches.
-            Loop { .. } => false,
-        }
-    }
-
-    /// Returns a reference to the [StackState] of the block.
-    pub fn stack_state(&self) -> &StackState {
-        use ControlStackFrame::*;
-        match self {
-            If { stack_state, .. }
-            | Else { stack_state, .. }
-            | Block { stack_state, .. }
-            | Loop { stack_state, .. } => stack_state,
-        }
-    }
-
-    /// Returns true if the current frame is [ControlStackFrame::If].
-    pub fn is_if(&self) -> bool {
-        match self {
-            Self::If { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Returns true if the current frame is [ControlStackFrame::Loop].
-    pub fn is_loop(&self) -> bool {
-        match self {
-            Self::Loop { .. } => true,
-            _ => false,
-        }
-    }
-}