wasmtime 10.0.1 → 11.0.0

data/ext/cargo-vendor/wasmtime-environ-11.0.0/src/fact/trampoline.rs

@@ -0,0 +1,3173 @@
+//! Low-level compilation of an fused adapter function.
+//!
+//! This module is tasked with the top-level `compile` function which creates a
+//! single WebAssembly function which will perform the steps of the fused
+//! adapter for an `AdapterData` provided. This is the "meat" of compilation
+//! where the validation of the canonical ABI or similar all happens to
+//! translate arguments from one module to another.
+//!
+//! ## Traps and their ordering
+//!
+//! Currently this compiler is pretty "loose" about the ordering of precisely
+//! what trap happens where. The main reason for this is that to core wasm all
+//! traps are the same and for fused adapters if a trap happens no intermediate
+//! side effects are visible (as designed by the canonical ABI itself). For this
+//! it's important to note that some of the precise choices of control flow here
+//! can be somewhat arbitrary, an intentional decision.
+
+use crate::component::{
+    CanonicalAbiInfo, ComponentTypesBuilder, FlatType, InterfaceType, StringEncoding,
+    TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, TypeRecordIndex,
+    TypeResultIndex, TypeTupleIndex, TypeUnionIndex, TypeVariantIndex, VariantInfo, FLAG_MAY_ENTER,
+    FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS,
+};
+use crate::fact::signature::Signature;
+use crate::fact::transcode::{FixedEncoding as FE, Transcode, Transcoder};
+use crate::fact::traps::Trap;
+use crate::fact::{
+    AdapterData, Body, Context, Function, FunctionId, Helper, HelperLocation, HelperType, Module,
+    Options,
+};
+use crate::{FuncIndex, GlobalIndex};
+use std::collections::HashMap;
+use std::mem;
+use std::ops::Range;
+use wasm_encoder::{BlockType, Encode, Instruction, Instruction::*, MemArg, ValType};
+use wasmtime_component_util::{DiscriminantSize, FlagsSize};
+
+const MAX_STRING_BYTE_LENGTH: u32 = 1 << 31;
+const UTF16_TAG: u32 = 1 << 31;
+
+/// This value is arbitrarily chosen and should be fine to change at any time,
+/// it just seemed like a halfway reasonable starting point.
+const INITIAL_FUEL: usize = 1_000;
+
+struct Compiler<'a, 'b> {
+    types: &'a ComponentTypesBuilder,
+    module: &'b mut Module<'a>,
+    result: FunctionId,
+
+    /// The encoded WebAssembly function body so far, not including locals.
+    code: Vec<u8>,
+
+    /// Total number of locals generated so far.
+    nlocals: u32,
+
+    /// Locals partitioned by type which are not currently in use.
+    free_locals: HashMap<ValType, Vec<u32>>,
+
+    /// Metadata about all `unreachable` trap instructions in this function and
+    /// what the trap represents. The offset within `self.code` is recorded as
+    /// well.
+    traps: Vec<(usize, Trap)>,
+
+    /// A heuristic which is intended to limit the size of a generated function
+    /// to a certain maximum to avoid generating arbitrarily large functions.
+    ///
+    /// This fuel counter is decremented each time `translate` is called and
+    /// when fuel is entirely consumed further translations, if necessary, will
+    /// be done through calls to other functions in the module. This is intended
+    /// to be a heuristic to split up the main function into theoretically
+    /// reusable portions.
+    fuel: usize,
+}
+
+pub(super) fn compile(module: &mut Module<'_>, adapter: &AdapterData) {
+    let lower_sig = module.types.signature(&adapter.lower, Context::Lower);
+    let lift_sig = module.types.signature(&adapter.lift, Context::Lift);
+    let ty = module
+        .core_types
+        .function(&lower_sig.params, &lower_sig.results);
+    let result = module
+        .funcs
+        .push(Function::new(Some(adapter.name.clone()), ty));
+    Compiler {
+        types: module.types,
+        module,
+        code: Vec::new(),
+        nlocals: lower_sig.params.len() as u32,
+        free_locals: HashMap::new(),
+        traps: Vec::new(),
+        result,
+        fuel: INITIAL_FUEL,
+    }
+    .compile_adapter(adapter, &lower_sig, &lift_sig)
+}
+
+/// Compiles a helper function as specified by the `Helper` configuration.
+///
+/// This function is invoked when the translation process runs out of fuel for
+/// some prior function which enqueues a helper to get translated later. This
+/// translation function will perform one type translation as specified by
+/// `Helper` which can either be in the stack or memory for each side.
+pub(super) fn compile_helper(module: &mut Module<'_>, result: FunctionId, helper: Helper) {
+    let mut nlocals = 0;
+    let src_flat;
+    let src = match helper.src.loc {
+        // If the source is on the stack then it's specified in the parameters
+        // to the function, so this creates the flattened representation and
+        // then lists those as the locals with appropriate types for the source
+        // values.
+        HelperLocation::Stack => {
+            src_flat = module
+                .types
+                .flatten_types(&helper.src.opts, usize::MAX, [helper.src.ty])
+                .unwrap()
+                .iter()
+                .enumerate()
+                .map(|(i, ty)| (i as u32, *ty))
+                .collect::<Vec<_>>();
+            nlocals += src_flat.len() as u32;
+            Source::Stack(Stack {
+                locals: &src_flat,
+                opts: &helper.src.opts,
+            })
+        }
+        // If the source is in memory then that's just propagated here as the
+        // first local is the pointer to the source.
+        HelperLocation::Memory => {
+            nlocals += 1;
+            Source::Memory(Memory {
+                opts: &helper.src.opts,
+                addr: TempLocal::new(0, helper.src.opts.ptr()),
+                offset: 0,
+            })
+        }
+    };
+    let dst_flat;
+    let dst = match helper.dst.loc {
+        // This is the same as the stack-based source although `Destination` is
+        // configured slightly differently.
+        HelperLocation::Stack => {
+            dst_flat = module
+                .types
+                .flatten_types(&helper.dst.opts, usize::MAX, [helper.dst.ty])
+                .unwrap();
+            Destination::Stack(&dst_flat, &helper.dst.opts)
+        }
+        // This is the same as a memroy-based source but note that the address
+        // of the destination is passed as the final parameter to the function.
+        HelperLocation::Memory => {
+            nlocals += 1;
+            Destination::Memory(Memory {
+                opts: &helper.dst.opts,
+                addr: TempLocal::new(nlocals - 1, helper.dst.opts.ptr()),
+                offset: 0,
+            })
+        }
+    };
+    let mut compiler = Compiler {
+        types: module.types,
+        module,
+        code: Vec::new(),
+        nlocals,
+        free_locals: HashMap::new(),
+        traps: Vec::new(),
+        result,
+        fuel: INITIAL_FUEL,
+    };
+    compiler.translate(&helper.src.ty, &src, &helper.dst.ty, &dst);
+    compiler.finish();
+}
+
+/// Possible ways that a interface value is represented in the core wasm
+/// canonical ABI.
+enum Source<'a> {
+    /// This value is stored on the "stack" in wasm locals.
+    ///
+    /// This could mean that it's inline from the parameters to the function or
+    /// that after a function call the results were stored in locals and the
+    /// locals are the inline results.
+    Stack(Stack<'a>),
+
+    /// This value is stored in linear memory described by the `Memory`
+    /// structure.
+    Memory(Memory<'a>),
+}
+
+/// Same as `Source` but for where values are translated into.
+enum Destination<'a> {
+    /// This value is destined for the WebAssembly stack which means that
+    /// results are simply pushed as we go along.
+    ///
+    /// The types listed are the types that are expected to be on the stack at
+    /// the end of translation.
+    Stack(&'a [ValType], &'a Options),
+
+    /// This value is to be placed in linear memory described by `Memory`.
+    Memory(Memory<'a>),
+}
+
+struct Stack<'a> {
+    /// The locals that comprise a particular value.
+    ///
+    /// The length of this list represents the flattened list of types that make
+    /// up the component value. Each list has the index of the local being
+    /// accessed as well as the type of the local itself.
+    locals: &'a [(u32, ValType)],
+    /// The lifting/lowering options for where this stack of values comes from
+    opts: &'a Options,
+}
+
+/// Representation of where a value is going to be stored in linear memory.
+struct Memory<'a> {
+    /// The lifting/lowering options with memory configuration
+    opts: &'a Options,
+    /// The index of the local that contains the base address of where the
+    /// storage is happening.
+    addr: TempLocal,
+    /// A "static" offset that will be baked into wasm instructions for where
+    /// memory loads/stores happen.
+    offset: u32,
+}
+
+impl Compiler<'_, '_> {
+    fn compile_adapter(
+        mut self,
+        adapter: &AdapterData,
+        lower_sig: &Signature,
+        lift_sig: &Signature,
+    ) {
+        // Check the instance flags required for this trampoline.
+        //
+        // This inserts the initial check required by `canon_lower` that the
+        // caller instance can be left and additionally checks the
+        // flags on the callee if necessary whether it can be entered.
+        self.trap_if_not_flag(adapter.lower.flags, FLAG_MAY_LEAVE, Trap::CannotLeave);
+        if adapter.called_as_export {
+            self.trap_if_not_flag(adapter.lift.flags, FLAG_MAY_ENTER, Trap::CannotEnter);
+            self.set_flag(adapter.lift.flags, FLAG_MAY_ENTER, false);
+        } else if self.module.debug {
+            self.assert_not_flag(
+                adapter.lift.flags,
+                FLAG_MAY_ENTER,
+                "may_enter should be unset",
+            );
+        }
+
+        // Perform the translation of arguments. Note that `FLAG_MAY_LEAVE` is
+        // cleared around this invocation for the callee as per the
+        // `canon_lift` definition in the spec. Additionally note that the
+        // precise ordering of traps here is not required since internal state
+        // is not visible to either instance and a trap will "lock down" both
+        // instances to no longer be visible. This means that we're free to
+        // reorder lifts/lowers and flags and such as is necessary and
+        // convenient here.
+        //
+        // TODO: if translation doesn't actually call any functions in either
+        // instance then there's no need to set/clear the flag here and that can
+        // be optimized away.
+        self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, false);
+        let param_locals = lower_sig
+            .params
+            .iter()
+            .enumerate()
+            .map(|(i, ty)| (i as u32, *ty))
+            .collect::<Vec<_>>();
+        self.translate_params(adapter, &param_locals);
+        self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, true);
+
+        // With all the arguments on the stack the actual target function is
+        // now invoked. The core wasm results of the function are then placed
+        // into locals for result translation afterwards.
+        self.instruction(Call(adapter.callee.as_u32()));
+        let mut result_locals = Vec::with_capacity(lift_sig.results.len());
+        let mut temps = Vec::new();
+        for ty in lift_sig.results.iter().rev() {
+            let local = self.local_set_new_tmp(*ty);
+            result_locals.push((local.idx, *ty));
+            temps.push(local);
+        }
+        result_locals.reverse();
+
+        // Like above during the translation of results the caller cannot be
+        // left (as we might invoke things like `realloc`). Again the precise
+        // order of everything doesn't matter since intermediate states cannot
+        // be witnessed, hence the setting of flags here to encapsulate both
+        // liftings and lowerings.
+        //
+        // TODO: like above the management of the `MAY_LEAVE` flag can probably
+        // be elided here for "simple" results.
+        self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, false);
+        self.translate_results(adapter, &param_locals, &result_locals);
+        self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, true);
+
+        // And finally post-return state is handled here once all results/etc
+        // are all translated.
+        if let Some(func) = adapter.lift.post_return {
+            for (result, _) in result_locals.iter() {
+                self.instruction(LocalGet(*result));
+            }
+            self.instruction(Call(func.as_u32()));
+        }
+        if adapter.called_as_export {
+            self.set_flag(adapter.lift.flags, FLAG_MAY_ENTER, true);
+        }
+
+        for tmp in temps {
+            self.free_temp_local(tmp);
+        }
+
+        self.finish()
+    }
+
+    fn translate_params(&mut self, adapter: &AdapterData, param_locals: &[(u32, ValType)]) {
+        let src_tys = self.types[adapter.lower.ty].params;
+        let src_tys = self.types[src_tys]
+            .types
+            .iter()
+            .copied()
+            .collect::<Vec<_>>();
+        let dst_tys = self.types[adapter.lift.ty].params;
+        let dst_tys = self.types[dst_tys]
+            .types
+            .iter()
+            .copied()
+            .collect::<Vec<_>>();
+        let lift_opts = &adapter.lift.options;
+        let lower_opts = &adapter.lower.options;
+
+        // TODO: handle subtyping
+        assert_eq!(src_tys.len(), dst_tys.len());
+
+        let src_flat =
+            self.types
+                .flatten_types(lower_opts, MAX_FLAT_PARAMS, src_tys.iter().copied());
+        let dst_flat =
+            self.types
+                .flatten_types(lift_opts, MAX_FLAT_PARAMS, dst_tys.iter().copied());
+
+        let src = if let Some(flat) = &src_flat {
+            Source::Stack(Stack {
+                locals: &param_locals[..flat.len()],
+                opts: lower_opts,
+            })
+        } else {
+            // If there are too many parameters then that means the parameters
+            // are actually a tuple stored in linear memory addressed by the
+            // first parameter local.
+            let (addr, ty) = param_locals[0];
+            assert_eq!(ty, lower_opts.ptr());
+            let align = src_tys
+                .iter()
+                .map(|t| self.types.align(lower_opts, t))
+                .max()
+                .unwrap_or(1);
+            Source::Memory(self.memory_operand(lower_opts, TempLocal::new(addr, ty), align))
+        };
+
+        let dst = if let Some(flat) = &dst_flat {
+            Destination::Stack(flat, lift_opts)
+        } else {
+            // If there are too many parameters then space is allocated in the
+            // destination module for the parameters via its `realloc` function.
+            let abi = CanonicalAbiInfo::record(dst_tys.iter().map(|t| self.types.canonical_abi(t)));
+            let (size, align) = if lift_opts.memory64 {
+                (abi.size64, abi.align64)
+            } else {
+                (abi.size32, abi.align32)
+            };
+            let size = MallocSize::Const(size);
+            Destination::Memory(self.malloc(lift_opts, size, align))
+        };
+
+        let srcs = src
+            .record_field_srcs(self.types, src_tys.iter().copied())
+            .zip(src_tys.iter());
+        let dsts = dst
+            .record_field_dsts(self.types, dst_tys.iter().copied())
+            .zip(dst_tys.iter());
+        for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) {
+            self.translate(&src_ty, &src, &dst_ty, &dst);
+        }
+
+        // If the destination was linear memory instead of the stack then the
+        // actual parameter that we're passing is the address of the values
+        // stored, so ensure that's happening in the wasm body here.
+        if let Destination::Memory(mem) = dst {
+            self.instruction(LocalGet(mem.addr.idx));
+            self.free_temp_local(mem.addr);
+        }
+    }
+
+    fn translate_results(
+        &mut self,
+        adapter: &AdapterData,
+        param_locals: &[(u32, ValType)],
+        result_locals: &[(u32, ValType)],
+    ) {
+        let src_tys = self.types[adapter.lift.ty].results;
+        let src_tys = self.types[src_tys]
+            .types
+            .iter()
+            .map(|ty| *ty)
+            .collect::<Vec<_>>();
+        let dst_tys = self.types[adapter.lower.ty].results;
+        let dst_tys = self.types[dst_tys]
+            .types
+            .iter()
+            .map(|ty| *ty)
+            .collect::<Vec<_>>();
+        let lift_opts = &adapter.lift.options;
+        let lower_opts = &adapter.lower.options;
+
+        let src_flat =
+            self.types
+                .flatten_types(lift_opts, MAX_FLAT_RESULTS, src_tys.iter().copied());
+        let dst_flat =
+            self.types
+                .flatten_types(lower_opts, MAX_FLAT_RESULTS, dst_tys.iter().copied());
+
+        let src = if src_flat.is_some() {
+            Source::Stack(Stack {
+                locals: result_locals,
+                opts: lift_opts,
+            })
+        } else {
+            // The original results to read from in this case come from the
+            // return value of the function itself. The imported function will
+            // return a linear memory address at which the values can be read
+            // from.
+            let align = src_tys
+                .iter()
+                .map(|t| self.types.align(lift_opts, t))
+                .max()
+                .unwrap_or(1);
+            assert_eq!(result_locals.len(), 1);
+            let (addr, ty) = result_locals[0];
+            assert_eq!(ty, lift_opts.ptr());
+            Source::Memory(self.memory_operand(lift_opts, TempLocal::new(addr, ty), align))
+        };
+
+        let dst = if let Some(flat) = &dst_flat {
+            Destination::Stack(flat, lower_opts)
+        } else {
+            // This is slightly different than `translate_params` where the
+            // return pointer was provided by the caller of this function
+            // meaning the last parameter local is a pointer into linear memory.
+            let align = dst_tys
+                .iter()
+                .map(|t| self.types.align(lower_opts, t))
+                .max()
+                .unwrap_or(1);
+            let (addr, ty) = *param_locals.last().expect("no retptr");
+            assert_eq!(ty, lower_opts.ptr());
+            Destination::Memory(self.memory_operand(lower_opts, TempLocal::new(addr, ty), align))
+        };
+
+        let srcs = src
+            .record_field_srcs(self.types, src_tys.iter().copied())
+            .zip(src_tys.iter());
+        let dsts = dst
+            .record_field_dsts(self.types, dst_tys.iter().copied())
+            .zip(dst_tys.iter());
+        for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) {
+            self.translate(&src_ty, &src, &dst_ty, &dst);
+        }
+    }
+
+    fn translate(
+        &mut self,
+        src_ty: &InterfaceType,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        if let Source::Memory(mem) = src {
+            self.assert_aligned(src_ty, mem);
+        }
+        if let Destination::Memory(mem) = dst {
+            self.assert_aligned(dst_ty, mem);
+        }
+
+        // Calculate a cost heuristic for what the translation of this specific
+        // layer of the type is going to incur. The purpose of this cost is that
+        // we'll deduct it from `self.fuel` and if no fuel is remaining then
+        // translation is outlined into a separate function rather than being
+        // translated into this function.
+        //
+        // The general goal is to avoid creating an exponentially sized function
+        // for a linearly sized input (the type section). By outlining helper
+        // functions there will ideally be a constant set of helper functions
+        // per type (to accomodate in-memory or on-stack transfers as well as
+        // src/dst options) which means that each function is at most a certain
+        // size and we have a linear number of functions which should guarantee
+        // an overall linear size of the output.
+        //
+        // To implement this the current heuristic is that each layer of
+        // translating a type has a cost associated with it and this cost is
+        // accounted for in `self.fuel`. Some conversions are considered free as
+        // they generate basically as much code as the `call` to the translation
+        // function while other are considered proportionally expensive to the
+        // size of the type. The hope is that some upper layers are of a type's
+        // translation are all inlined into one function but bottom layers end
+        // up getting outlined to separate functions. Theoretically, again this
+        // is built on hopes and dreams, the outlining can be shared amongst
+        // tightly-intertwined type hierarchies which will reduce the size of
+        // the output module due to the helpers being used.
+        //
+        // This heuristic of how to split functions has changed a few times in
+        // the past and this isn't necessarily guaranteed to be the final
+        // iteration.
+        let cost = match src_ty {
+            // These types are all quite simple to load/store and equate to
+            // basically the same cost of the `call` instruction to call an
+            // out-of-line translation function, so give them 0 cost.
+            InterfaceType::Bool
+            | InterfaceType::U8
+            | InterfaceType::S8
+            | InterfaceType::U16
+            | InterfaceType::S16
+            | InterfaceType::U32
+            | InterfaceType::S32
+            | InterfaceType::U64
+            | InterfaceType::S64
+            | InterfaceType::Float32
+            | InterfaceType::Float64 => 0,
+
+            // This has a small amount of validation associated with it, so
+            // give it a cost of 1.
+            InterfaceType::Char => 1,
+
+            // This has a fair bit of code behind it depending on the
+            // strings/encodings in play, so arbitrarily assign it this cost.
+            InterfaceType::String => 40,
+
+            // Iteration of a loop is along the lines of the cost of a string
+            // so give it the same cost
+            InterfaceType::List(_) => 40,
+
+            InterfaceType::Flags(i) => {
+                let count = self.module.types[*i].names.len();
+                match FlagsSize::from_count(count) {
+                    FlagsSize::Size0 => 0,
+                    FlagsSize::Size1 | FlagsSize::Size2 => 1,
+                    FlagsSize::Size4Plus(n) => n.into(),
+                }
+            }
+
+            InterfaceType::Record(i) => self.types[*i].fields.len(),
+            InterfaceType::Tuple(i) => self.types[*i].types.len(),
+            InterfaceType::Variant(i) => self.types[*i].cases.len(),
+            InterfaceType::Union(i) => self.types[*i].types.len(),
+            InterfaceType::Enum(i) => self.types[*i].names.len(),
+
+            // 2 cases to consider for each of these variants.
+            InterfaceType::Option(_) | InterfaceType::Result(_) => 2,
+        };
+
+        match self.fuel.checked_sub(cost) {
+            // This function has enough fuel to perform the layer of translation
+            // necessary for this type, so the fuel is updated in-place and
+            // translation continues. Note that the recursion here is bounded by
+            // the static recursion limit for all interface types as imposed
+            // during the translation phase.
+            Some(n) => {
+                self.fuel = n;
+                match src_ty {
+                    InterfaceType::Bool => self.translate_bool(src, dst_ty, dst),
+                    InterfaceType::U8 => self.translate_u8(src, dst_ty, dst),
+                    InterfaceType::S8 => self.translate_s8(src, dst_ty, dst),
+                    InterfaceType::U16 => self.translate_u16(src, dst_ty, dst),
+                    InterfaceType::S16 => self.translate_s16(src, dst_ty, dst),
+                    InterfaceType::U32 => self.translate_u32(src, dst_ty, dst),
+                    InterfaceType::S32 => self.translate_s32(src, dst_ty, dst),
+                    InterfaceType::U64 => self.translate_u64(src, dst_ty, dst),
+                    InterfaceType::S64 => self.translate_s64(src, dst_ty, dst),
+                    InterfaceType::Float32 => self.translate_f32(src, dst_ty, dst),
+                    InterfaceType::Float64 => self.translate_f64(src, dst_ty, dst),
+                    InterfaceType::Char => self.translate_char(src, dst_ty, dst),
+                    InterfaceType::String => self.translate_string(src, dst_ty, dst),
+                    InterfaceType::List(t) => self.translate_list(*t, src, dst_ty, dst),
+                    InterfaceType::Record(t) => self.translate_record(*t, src, dst_ty, dst),
+                    InterfaceType::Flags(f) => self.translate_flags(*f, src, dst_ty, dst),
+                    InterfaceType::Tuple(t) => self.translate_tuple(*t, src, dst_ty, dst),
+                    InterfaceType::Variant(v) => self.translate_variant(*v, src, dst_ty, dst),
+                    InterfaceType::Union(u) => self.translate_union(*u, src, dst_ty, dst),
+                    InterfaceType::Enum(t) => self.translate_enum(*t, src, dst_ty, dst),
+                    InterfaceType::Option(t) => self.translate_option(*t, src, dst_ty, dst),
+                    InterfaceType::Result(t) => self.translate_result(*t, src, dst_ty, dst),
+                }
+            }
+
+            // This function does not have enough fuel left to perform this
+            // layer of translation so the translation is deferred to a helper
+            // function. The actual translation here is then done by marshalling
+            // the src/dst into the function we're calling and then processing
+            // the results.
+            None => {
+                let src_loc = match src {
+                    // If the source is on the stack then `stack_get` is used to
+                    // convert everything to the appropriate flat representation
+                    // for the source type.
+                    Source::Stack(stack) => {
+                        for (i, ty) in stack
+                            .opts
+                            .flat_types(src_ty, self.types)
+                            .unwrap()
+                            .iter()
+                            .enumerate()
+                        {
+                            let stack = stack.slice(i..i + 1);
+                            self.stack_get(&stack, (*ty).into());
+                        }
+                        HelperLocation::Stack
+                    }
+                    // If the source is in memory then the pointer is passed
+                    // through, but note that the offset must be factored in
+                    // here since the translation function will start from
+                    // offset 0.
+                    Source::Memory(mem) => {
+                        self.push_mem_addr(mem);
+                        HelperLocation::Memory
+                    }
+                };
+                let dst_loc = match dst {
+                    Destination::Stack(..) => HelperLocation::Stack,
+                    Destination::Memory(mem) => {
+                        self.push_mem_addr(mem);
+                        HelperLocation::Memory
+                    }
+                };
+                // Generate a `FunctionId` corresponding to the `Helper`
+                // configuration that is necessary here. This will ideally be a
+                // "cache hit" and use a preexisting helper which represents
+                // outlining what would otherwise be duplicate code within a
+                // function to one function.
+                let helper = self.module.translate_helper(Helper {
+                    src: HelperType {
+                        ty: *src_ty,
+                        opts: *src.opts(),
+                        loc: src_loc,
+                    },
+                    dst: HelperType {
+                        ty: *dst_ty,
+                        opts: *dst.opts(),
+                        loc: dst_loc,
+                    },
+                });
+                // Emit a `call` instruction which will get "relocated" to a
+                // function index once translation has completely finished.
+                self.flush_code();
+                self.module.funcs[self.result].body.push(Body::Call(helper));
+
+                // If the destination of the translation was on the stack then
+                // the types on the stack need to be optionally converted to
+                // different types (e.g. if the result here is part of a variant
+                // somewhere else).
+                //
+                // This translation happens inline here by popping the results
+                // into new locals and then using those locals to do a
+                // `stack_set`.
+                if let Destination::Stack(tys, opts) = dst {
+                    let flat = self
+                        .types
+                        .flatten_types(opts, usize::MAX, [*dst_ty])
+                        .unwrap();
+                    assert_eq!(flat.len(), tys.len());
+                    let locals = flat
+                        .iter()
+                        .rev()
+                        .map(|ty| self.local_set_new_tmp(*ty))
+                        .collect::<Vec<_>>();
+                    for (ty, local) in tys.iter().zip(locals.into_iter().rev()) {
+                        self.instruction(LocalGet(local.idx));
+                        self.stack_set(std::slice::from_ref(ty), local.ty);
+                        self.free_temp_local(local);
+                    }
+                }
+            }
+        }
+    }
+
+    fn push_mem_addr(&mut self, mem: &Memory<'_>) {
+        self.instruction(LocalGet(mem.addr.idx));
+        if mem.offset != 0 {
+            self.ptr_uconst(mem.opts, mem.offset);
+            self.ptr_add(mem.opts);
+        }
+    }
+
+    fn translate_bool(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::Bool));
+        self.push_dst_addr(dst);
+
+        // Booleans are canonicalized to 0 or 1 as they pass through the
+        // component boundary, so use a `select` instruction to do so.
+        self.instruction(I32Const(1));
+        self.instruction(I32Const(0));
+        match src {
+            Source::Memory(mem) => self.i32_load8u(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::I32),
+        }
+        self.instruction(Select);
+
+        match dst {
+            Destination::Memory(mem) => self.i32_store8(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_u8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::U8));
+        self.convert_u8_mask(src, dst, 0xff);
+    }
+
+    fn convert_u8_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u8) {
+        self.push_dst_addr(dst);
+        let mut needs_mask = true;
+        match src {
+            Source::Memory(mem) => {
+                self.i32_load8u(mem);
+                needs_mask = mask != 0xff;
+            }
+            Source::Stack(stack) => {
+                self.stack_get(stack, ValType::I32);
+            }
+        }
+        if needs_mask {
+            self.instruction(I32Const(i32::from(mask)));
+            self.instruction(I32And);
+        }
+        match dst {
+            Destination::Memory(mem) => self.i32_store8(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_s8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::S8));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.i32_load8s(mem),
+            Source::Stack(stack) => {
+                self.stack_get(stack, ValType::I32);
+                self.instruction(I32Extend8S);
+            }
+        }
+        match dst {
+            Destination::Memory(mem) => self.i32_store8(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_u16(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::U16));
+        self.convert_u16_mask(src, dst, 0xffff);
+    }
+
+    fn convert_u16_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u16) {
+        self.push_dst_addr(dst);
+        let mut needs_mask = true;
+        match src {
+            Source::Memory(mem) => {
+                self.i32_load16u(mem);
+                needs_mask = mask != 0xffff;
+            }
+            Source::Stack(stack) => {
+                self.stack_get(stack, ValType::I32);
+            }
+        }
+        if needs_mask {
+            self.instruction(I32Const(i32::from(mask)));
+            self.instruction(I32And);
+        }
+        match dst {
+            Destination::Memory(mem) => self.i32_store16(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_s16(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::S16));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.i32_load16s(mem),
+            Source::Stack(stack) => {
+                self.stack_get(stack, ValType::I32);
+                self.instruction(I32Extend16S);
+            }
+        }
+        match dst {
+            Destination::Memory(mem) => self.i32_store16(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_u32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::U32));
+        self.convert_u32_mask(src, dst, 0xffffffff)
+    }
+
+    fn convert_u32_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u32) {
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.i32_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::I32),
+        }
+        if mask != 0xffffffff {
+            self.instruction(I32Const(mask as i32));
+            self.instruction(I32And);
+        }
+        match dst {
+            Destination::Memory(mem) => self.i32_store(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_s32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::S32));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.i32_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::I32),
+        }
+        match dst {
+            Destination::Memory(mem) => self.i32_store(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+    }
+
+    fn translate_u64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::U64));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.i64_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::I64),
+        }
+        match dst {
+            Destination::Memory(mem) => self.i64_store(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I64),
+        }
+    }
+
+    fn translate_s64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::S64));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.i64_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::I64),
+        }
+        match dst {
+            Destination::Memory(mem) => self.i64_store(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I64),
+        }
+    }
+
+    fn translate_f32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::Float32));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.f32_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::F32),
+        }
+        match dst {
+            Destination::Memory(mem) => self.f32_store(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::F32),
+        }
+    }
+
+    fn translate_f64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        // TODO: subtyping
+        assert!(matches!(dst_ty, InterfaceType::Float64));
+        self.push_dst_addr(dst);
+        match src {
+            Source::Memory(mem) => self.f64_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::F64),
+        }
+        match dst {
+            Destination::Memory(mem) => self.f64_store(mem),
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::F64),
+        }
+    }
+
+    fn translate_char(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        assert!(matches!(dst_ty, InterfaceType::Char));
+        match src {
+            Source::Memory(mem) => self.i32_load(mem),
+            Source::Stack(stack) => self.stack_get(stack, ValType::I32),
+        }
+        let local = self.local_set_new_tmp(ValType::I32);
+
+        // This sequence is copied from the output of LLVM for:
+        //
+        //      pub extern "C" fn foo(x: u32) -> char {
+        //          char::try_from(x)
+        //              .unwrap_or_else(|_| std::arch::wasm32::unreachable())
+        //      }
+        //
+        // Apparently this does what's required by the canonical ABI:
+        //
+        //    def i32_to_char(opts, i):
+        //      trap_if(i >= 0x110000)
+        //      trap_if(0xD800 <= i <= 0xDFFF)
+        //      return chr(i)
+        //
+        // ... but I don't know how it works other than "well I trust LLVM"
+        self.instruction(Block(BlockType::Empty));
+        self.instruction(Block(BlockType::Empty));
+        self.instruction(LocalGet(local.idx));
+        self.instruction(I32Const(0xd800));
+        self.instruction(I32Xor);
+        self.instruction(I32Const(-0x110000));
+        self.instruction(I32Add);
+        self.instruction(I32Const(-0x10f800));
+        self.instruction(I32LtU);
+        self.instruction(BrIf(0));
+        self.instruction(LocalGet(local.idx));
+        self.instruction(I32Const(0x110000));
+        self.instruction(I32Ne);
+        self.instruction(BrIf(1));
+        self.instruction(End);
+        self.trap(Trap::InvalidChar);
+        self.instruction(End);
+
+        self.push_dst_addr(dst);
+        self.instruction(LocalGet(local.idx));
+        match dst {
+            Destination::Memory(mem) => {
+                self.i32_store(mem);
+            }
+            Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
+        }
+
+        self.free_temp_local(local);
+    }
+
+    fn translate_string(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
+        assert!(matches!(dst_ty, InterfaceType::String));
+        let src_opts = src.opts();
+        let dst_opts = dst.opts();
+
+        // Load the pointer/length of this string into temporary locals. These
+        // will be referenced a good deal so this just makes it easier to deal
+        // with them consistently below rather than trying to reload from memory
+        // for example.
+        match src {
+            Source::Stack(s) => {
+                assert_eq!(s.locals.len(), 2);
+                self.stack_get(&s.slice(0..1), src_opts.ptr());
+                self.stack_get(&s.slice(1..2), src_opts.ptr());
+            }
+            Source::Memory(mem) => {
+                self.ptr_load(mem);
+                self.ptr_load(&mem.bump(src_opts.ptr_size().into()));
+            }
+        }
+        let src_len = self.local_set_new_tmp(src_opts.ptr());
+        let src_ptr = self.local_set_new_tmp(src_opts.ptr());
+        let src_str = WasmString {
+            ptr: src_ptr,
+            len: src_len,
+            opts: src_opts,
+        };
+
+        let dst_str = match src_opts.string_encoding {
+            StringEncoding::Utf8 => match dst_opts.string_encoding {
+                StringEncoding::Utf8 => self.string_copy(&src_str, FE::Utf8, dst_opts, FE::Utf8),
+                StringEncoding::Utf16 => self.string_utf8_to_utf16(&src_str, dst_opts),
+                StringEncoding::CompactUtf16 => {
+                    self.string_to_compact(&src_str, FE::Utf8, dst_opts)
+                }
+            },
+
+            StringEncoding::Utf16 => {
+                self.verify_aligned(src_opts, src_str.ptr.idx, 2);
+                match dst_opts.string_encoding {
+                    StringEncoding::Utf8 => {
+                        self.string_deflate_to_utf8(&src_str, FE::Utf16, dst_opts)
+                    }
+                    StringEncoding::Utf16 => {
+                        self.string_copy(&src_str, FE::Utf16, dst_opts, FE::Utf16)
+                    }
+                    StringEncoding::CompactUtf16 => {
+                        self.string_to_compact(&src_str, FE::Utf16, dst_opts)
+                    }
+                }
+            }
+
+            StringEncoding::CompactUtf16 => {
+                self.verify_aligned(src_opts, src_str.ptr.idx, 2);
+
+                // Test the tag big to see if this is a utf16 or a latin1 string
+                // at runtime...
+                self.instruction(LocalGet(src_str.len.idx));
+                self.ptr_uconst(src_opts, UTF16_TAG);
+                self.ptr_and(src_opts);
+                self.ptr_if(src_opts, BlockType::Empty);
+
+                // In the utf16 block unset the upper bit from the length local
+                // so further calculations have the right value. Afterwards the
+                // string transcode proceeds assuming utf16.
+                self.instruction(LocalGet(src_str.len.idx));
+                self.ptr_uconst(src_opts, UTF16_TAG);
+                self.ptr_xor(src_opts);
+                self.instruction(LocalSet(src_str.len.idx));
+                let s1 = match dst_opts.string_encoding {
+                    StringEncoding::Utf8 => {
+                        self.string_deflate_to_utf8(&src_str, FE::Utf16, dst_opts)
+                    }
+                    StringEncoding::Utf16 => {
+                        self.string_copy(&src_str, FE::Utf16, dst_opts, FE::Utf16)
+                    }
+                    StringEncoding::CompactUtf16 => {
+                        self.string_compact_utf16_to_compact(&src_str, dst_opts)
+                    }
+                };
+
+                self.instruction(Else);
+
+                // In the latin1 block the `src_len` local is already the number
+                // of code units, so the string transcoding is all that needs to
+                // happen.
+                let s2 = match dst_opts.string_encoding {
+                    StringEncoding::Utf16 => {
+                        self.string_copy(&src_str, FE::Latin1, dst_opts, FE::Utf16)
+                    }
+                    StringEncoding::Utf8 => {
+                        self.string_deflate_to_utf8(&src_str, FE::Latin1, dst_opts)
+                    }
+                    StringEncoding::CompactUtf16 => {
+                        self.string_copy(&src_str, FE::Latin1, dst_opts, FE::Latin1)
+                    }
+                };
+                // Set our `s2` generated locals to the `s2` generated locals
+                // as the resulting pointer of this transcode.
+                self.instruction(LocalGet(s2.ptr.idx));
+                self.instruction(LocalSet(s1.ptr.idx));
+                self.instruction(LocalGet(s2.len.idx));
+                self.instruction(LocalSet(s1.len.idx));
+                self.instruction(End);
+                self.free_temp_local(s2.ptr);
+                self.free_temp_local(s2.len);
+                s1
+            }
+        };
+
+        // Store the ptr/length in the desired destination
+        match dst {
+            Destination::Stack(s, _) => {
+                self.instruction(LocalGet(dst_str.ptr.idx));
+                self.stack_set(&s[..1], dst_opts.ptr());
+                self.instruction(LocalGet(dst_str.len.idx));
+                self.stack_set(&s[1..], dst_opts.ptr());
+            }
+            Destination::Memory(mem) => {
+                self.instruction(LocalGet(mem.addr.idx));
+                self.instruction(LocalGet(dst_str.ptr.idx));
+                self.ptr_store(mem);
+                self.instruction(LocalGet(mem.addr.idx));
+                self.instruction(LocalGet(dst_str.len.idx));
+                self.ptr_store(&mem.bump(dst_opts.ptr_size().into()));
+            }
+        }
+
+        self.free_temp_local(src_str.ptr);
+        self.free_temp_local(src_str.len);
+        self.free_temp_local(dst_str.ptr);
+        self.free_temp_local(dst_str.len);
+    }
+
+    // Corresponding function for `store_string_copy` in the spec.
+    //
+    // This performs a transcoding of the string with a one-pass copy from
+    // the `src` encoding to the `dst` encoding. This is only possible for
+    // fixed encodings where the first allocation is guaranteed to be an
+    // appropriate fit so it's not suitable for all encodings.
+    //
+    // Imported host transcoding functions here take the src/dst pointers as
+    // well as the number of code units in the source (which always matches
+    // the number of code units in the destination). There is no return
+    // value from the transcode function since the encoding should always
+    // work on the first pass.
+    fn string_copy<'a>(
+        &mut self,
+        src: &WasmString<'_>,
+        src_enc: FE,
+        dst_opts: &'a Options,
+        dst_enc: FE,
+    ) -> WasmString<'a> {
+        assert!(dst_enc.width() >= src_enc.width());
+        self.validate_string_length(src, dst_enc);
+
+        // Calculate the source byte length given the size of each code
+        // unit. Note that this shouldn't overflow given
+        // `validate_string_length` above.
+        let mut src_byte_len_tmp = None;
+        let src_byte_len = if src_enc.width() == 1 {
+            src.len.idx
+        } else {
+            assert_eq!(src_enc.width(), 2);
+            self.instruction(LocalGet(src.len.idx));
+            self.ptr_uconst(src.opts, 1);
+            self.ptr_shl(src.opts);
+            let tmp = self.local_set_new_tmp(src.opts.ptr());
+            let ret = tmp.idx;
+            src_byte_len_tmp = Some(tmp);
+            ret
+        };
+
+        // Convert the source code units length to the destination byte
+        // length type.
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
+        let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
+        if dst_enc.width() > 1 {
+            assert_eq!(dst_enc.width(), 2);
+            self.ptr_uconst(dst_opts, 1);
+            self.ptr_shl(dst_opts);
+        }
+        let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
+
+        // Allocate space in the destination using the calculated byte
+        // length.
+        let dst = {
+            let dst_mem = self.malloc(
+                dst_opts,
+                MallocSize::Local(dst_byte_len.idx),
+                dst_enc.width().into(),
+            );
+            WasmString {
+                ptr: dst_mem.addr,
+                len: dst_len,
+                opts: dst_opts,
+            }
+        };
+
+        // Validate that `src_len + src_ptr` and
+        // `dst_mem.addr_local + dst_byte_len` are both in-bounds. This
+        // is done by loading the last byte of the string and if that
+        // doesn't trap then it's known valid.
+        self.validate_string_inbounds(src, src_byte_len);
+        self.validate_string_inbounds(&dst, dst_byte_len.idx);
+
+        // If the validations pass then the host `transcode` intrinsic
+        // is invoked. This will either raise a trap or otherwise succeed
+        // in which case we're done.
+        let op = if src_enc == dst_enc {
+            Transcode::Copy(src_enc)
+        } else {
+            assert_eq!(src_enc, FE::Latin1);
+            assert_eq!(dst_enc, FE::Utf16);
+            Transcode::Latin1ToUtf16
+        };
+        let transcode = self.transcoder(src, &dst, op);
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(Call(transcode.as_u32()));
+
+        self.free_temp_local(dst_byte_len);
+        if let Some(tmp) = src_byte_len_tmp {
+            self.free_temp_local(tmp);
+        }
+
+        dst
+    }
+    // Corresponding function for `store_string_to_utf8` in the spec.
+    //
+    // This translation works by possibly performing a number of
+    // reallocations. First a buffer of size input-code-units is used to try
+    // to get the transcoding correct on the first try. If that fails the
+    // maximum worst-case size is used and then that is resized down if it's
+    // too large.
+    //
+    // The host transcoding function imported here will receive src ptr/len
+    // and dst ptr/len and return how many code units were consumed on both
+    // sides. The amount of code units consumed in the source dictates which
+    // branches are taken in this conversion.
+    fn string_deflate_to_utf8<'a>(
+        &mut self,
+        src: &WasmString<'_>,
+        src_enc: FE,
+        dst_opts: &'a Options,
+    ) -> WasmString<'a> {
+        self.validate_string_length(src, src_enc);
+
+        // Optimistically assume that the code unit length of the source is
+        // all that's needed in the destination. Perform that allocaiton
+        // here and proceed to transcoding below.
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
+        let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
+        let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
+
+        let dst = {
+            let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 1);
+            WasmString {
+                ptr: dst_mem.addr,
+                len: dst_len,
+                opts: dst_opts,
+            }
+        };
+
+        // Ensure buffers are all in-bounds
+        let mut src_byte_len_tmp = None;
+        let src_byte_len = match src_enc {
+            FE::Latin1 => src.len.idx,
+            FE::Utf16 => {
+                self.instruction(LocalGet(src.len.idx));
+                self.ptr_uconst(src.opts, 1);
+                self.ptr_shl(src.opts);
+                let tmp = self.local_set_new_tmp(src.opts.ptr());
+                let ret = tmp.idx;
+                src_byte_len_tmp = Some(tmp);
+                ret
+            }
+            FE::Utf8 => unreachable!(),
+        };
+        self.validate_string_inbounds(src, src_byte_len);
+        self.validate_string_inbounds(&dst, dst_byte_len.idx);
+
+        // Perform the initial transcode
+        let op = match src_enc {
+            FE::Latin1 => Transcode::Latin1ToUtf8,
+            FE::Utf16 => Transcode::Utf16ToUtf8,
+            FE::Utf8 => unreachable!(),
+        };
+        let transcode = self.transcoder(src, &dst, op);
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(LocalGet(dst_byte_len.idx));
+        self.instruction(Call(transcode.as_u32()));
+        self.instruction(LocalSet(dst.len.idx));
+        let src_len_tmp = self.local_set_new_tmp(src.opts.ptr());
+
+        // Test if the source was entirely transcoded by comparing
+        // `src_len_tmp`, the number of code units transcoded from the
+        // source, with `src_len`, the original number of code units.
+        self.instruction(LocalGet(src_len_tmp.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.ptr_ne(src.opts);
+        self.instruction(If(BlockType::Empty));
+
+        // Here a worst-case reallocation is performed to grow `dst_mem`.
+        // In-line a check is also performed that the worst-case byte size
+        // fits within the maximum size of strings.
+        self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
+        self.instruction(LocalGet(dst_byte_len.idx)); // old_size
+        self.ptr_uconst(dst.opts, 1); // align
+        let factor = match src_enc {
+            FE::Latin1 => 2,
+            FE::Utf16 => 3,
+            _ => unreachable!(),
+        };
+        self.validate_string_length_u8(src, factor);
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
+        self.ptr_uconst(dst_opts, factor.into());
+        self.ptr_mul(dst_opts);
+        self.instruction(LocalTee(dst_byte_len.idx));
+        self.instruction(Call(dst_opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+
+        // Verify that the destination is still in-bounds
+        self.validate_string_inbounds(&dst, dst_byte_len.idx);
+
+        // Perform another round of transcoding that should be guaranteed
+        // to succeed. Note that all the parameters here are offset by the
+        // results of the first transcoding to only perform the remaining
+        // transcode on the final units.
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src_len_tmp.idx));
+        if let FE::Utf16 = src_enc {
+            self.ptr_uconst(src.opts, 1);
+            self.ptr_shl(src.opts);
+        }
+        self.ptr_add(src.opts);
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(src_len_tmp.idx));
+        self.ptr_sub(src.opts);
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_add(dst.opts);
+        self.instruction(LocalGet(dst_byte_len.idx));
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_sub(dst.opts);
+        self.instruction(Call(transcode.as_u32()));
+
+        // Add the second result, the amount of destination units encoded,
+        // to `dst_len` so it's an accurate reflection of the final size of
+        // the destination buffer.
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_add(dst.opts);
+        self.instruction(LocalSet(dst.len.idx));
+
+        // In debug mode verify the first result consumed the entire string,
+        // otherwise simply discard it.
+        if self.module.debug {
+            self.instruction(LocalGet(src.len.idx));
+            self.instruction(LocalGet(src_len_tmp.idx));
+            self.ptr_sub(src.opts);
+            self.ptr_ne(src.opts);
+            self.instruction(If(BlockType::Empty));
+            self.trap(Trap::AssertFailed("should have finished encoding"));
+            self.instruction(End);
+        } else {
+            self.instruction(Drop);
+        }
+
+        // Perform a downsizing if the worst-case size was too large
+        self.instruction(LocalGet(dst.len.idx));
+        self.instruction(LocalGet(dst_byte_len.idx));
+        self.ptr_ne(dst.opts);
+        self.instruction(If(BlockType::Empty));
+        self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
+        self.instruction(LocalGet(dst_byte_len.idx)); // old_size
+        self.ptr_uconst(dst.opts, 1); // align
+        self.instruction(LocalGet(dst.len.idx)); // new_size
+        self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+        self.instruction(End);
+
+        // If the first transcode was enough then assert that the returned
+        // amount of destination items written equals the byte size.
+        if self.module.debug {
+            self.instruction(Else);
+
+            self.instruction(LocalGet(dst.len.idx));
+            self.instruction(LocalGet(dst_byte_len.idx));
+            self.ptr_ne(dst_opts);
+            self.instruction(If(BlockType::Empty));
+            self.trap(Trap::AssertFailed("should have finished encoding"));
+            self.instruction(End);
+        }
+
+        self.instruction(End); // end of "first transcode not enough"
+
+        self.free_temp_local(src_len_tmp);
+        self.free_temp_local(dst_byte_len);
+        if let Some(tmp) = src_byte_len_tmp {
+            self.free_temp_local(tmp);
+        }
+
+        dst
+    }
+
+    // Corresponds to the `store_utf8_to_utf16` function in the spec.
+    //
+    // When converting utf-8 to utf-16 a pessimistic allocation is
+    // done which is twice the byte length of the utf-8 string.
+    // The host then transcodes and returns how many code units were
+    // actually used during the transcoding and if it's beneath the
+    // pessimistic maximum then the buffer is reallocated down to
+    // a smaller amount.
+    //
+    // The host-imported transcoding function takes the src/dst pointer as
+    // well as the code unit size of both the source and destination. The
+    // destination should always be big enough to hold the result of the
+    // transcode and so the result of the host function is how many code
+    // units were written to the destination.
+    fn string_utf8_to_utf16<'a>(
+        &mut self,
+        src: &WasmString<'_>,
+        dst_opts: &'a Options,
+    ) -> WasmString<'a> {
+        self.validate_string_length(src, FE::Utf16);
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
+        let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
+        self.ptr_uconst(dst_opts, 1);
+        self.ptr_shl(dst_opts);
+        let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
+        let dst = {
+            let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2);
+            WasmString {
+                ptr: dst_mem.addr,
+                len: dst_len,
+                opts: dst_opts,
+            }
+        };
+
+        self.validate_string_inbounds(src, src.len.idx);
+        self.validate_string_inbounds(&dst, dst_byte_len.idx);
+
+        let transcode = self.transcoder(src, &dst, Transcode::Utf8ToUtf16);
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(Call(transcode.as_u32()));
+        self.instruction(LocalSet(dst.len.idx));
+
+        // If the number of code units returned by transcode is not
+        // equal to the original number of code units then
+        // the buffer must be shrunk.
+        //
+        // Note that the byte length of the final allocation we
+        // want is twice the code unit length returned by the
+        // transcoding function.
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_ne(dst_opts);
+        self.instruction(If(BlockType::Empty));
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(LocalGet(dst_byte_len.idx));
+        self.ptr_uconst(dst.opts, 2);
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_uconst(dst.opts, 1);
+        self.ptr_shl(dst.opts);
+        self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+        self.instruction(End); // end of shrink-to-fit
+
+        self.free_temp_local(dst_byte_len);
+
+        dst
+    }
+
+    // Corresponds to `store_probably_utf16_to_latin1_or_utf16` in the spec.
+    //
+    // This will try to transcode the input utf16 string to utf16 in the
+    // destination. If utf16 isn't needed though and latin1 could be used
+    // then that's used instead and a reallocation to downsize occurs
+    // afterwards.
+    //
+    // The host transcode function here will take the src/dst pointers as
+    // well as src length. The destination byte length is twice the src code
+    // unit length. The return value is the tagged length of the returned
+    // string. If the upper bit is set then utf16 was used and the
+    // conversion is done. If the upper bit is not set then latin1 was used
+    // and a downsizing needs to happen.
+    fn string_compact_utf16_to_compact<'a>(
+        &mut self,
+        src: &WasmString<'_>,
+        dst_opts: &'a Options,
+    ) -> WasmString<'a> {
+        self.validate_string_length(src, FE::Utf16);
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
+        let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
+        self.ptr_uconst(dst_opts, 1);
+        self.ptr_shl(dst_opts);
+        let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
+        let dst = {
+            let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2);
+            WasmString {
+                ptr: dst_mem.addr,
+                len: dst_len,
+                opts: dst_opts,
+            }
+        };
+
+        self.convert_src_len_to_dst(dst_byte_len.idx, dst.opts.ptr(), src.opts.ptr());
+        let src_byte_len = self.local_set_new_tmp(src.opts.ptr());
+
+        self.validate_string_inbounds(src, src_byte_len.idx);
+        self.validate_string_inbounds(&dst, dst_byte_len.idx);
+
+        let transcode = self.transcoder(src, &dst, Transcode::Utf16ToCompactProbablyUtf16);
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(Call(transcode.as_u32()));
+        self.instruction(LocalSet(dst.len.idx));
+
+        // Assert that the untagged code unit length is the same as the
+        // source code unit length.
+        if self.module.debug {
+            self.instruction(LocalGet(dst.len.idx));
+            self.ptr_uconst(dst.opts, !UTF16_TAG);
+            self.ptr_and(dst.opts);
+            self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
+            self.ptr_ne(dst.opts);
+            self.instruction(If(BlockType::Empty));
+            self.trap(Trap::AssertFailed("expected equal code units"));
+            self.instruction(End);
+        }
+
+        // If the UTF16_TAG is set then utf16 was used and the destination
+        // should be appropriately sized. Bail out of the "is this string
+        // empty" block and fall through otherwise to resizing.
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_uconst(dst.opts, UTF16_TAG);
+        self.ptr_and(dst.opts);
+        self.ptr_br_if(dst.opts, 0);
+
+        // Here `realloc` is used to downsize the string
+        self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
+        self.instruction(LocalGet(dst_byte_len.idx)); // old_size
+        self.ptr_uconst(dst.opts, 2); // align
+        self.instruction(LocalGet(dst.len.idx)); // new_size
+        self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+
+        self.free_temp_local(dst_byte_len);
+        self.free_temp_local(src_byte_len);
+
+        dst
+    }
+
+    // Corresponds to `store_string_to_latin1_or_utf16` in the spec.
+    //
+    // This will attempt a first pass of transcoding to latin1 and on
+    // failure a larger buffer is allocated for utf16 and then utf16 is
+    // encoded in-place into the buffer. After either latin1 or utf16 the
+    // buffer is then resized to fit the final string allocation.
+    fn string_to_compact<'a>(
+        &mut self,
+        src: &WasmString<'_>,
+        src_enc: FE,
+        dst_opts: &'a Options,
+    ) -> WasmString<'a> {
+        self.validate_string_length(src, src_enc);
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
+        let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
+        let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
+        let dst = {
+            let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2);
+            WasmString {
+                ptr: dst_mem.addr,
+                len: dst_len,
+                opts: dst_opts,
+            }
+        };
+
+        self.validate_string_inbounds(src, src.len.idx);
+        self.validate_string_inbounds(&dst, dst_byte_len.idx);
+
+        // Perform the initial latin1 transcode. This returns the number of
+        // source code units consumed and the number of destination code
+        // units (bytes) written.
+        let (latin1, utf16) = match src_enc {
+            FE::Utf8 => (Transcode::Utf8ToLatin1, Transcode::Utf8ToCompactUtf16),
+            FE::Utf16 => (Transcode::Utf16ToLatin1, Transcode::Utf16ToCompactUtf16),
+            FE::Latin1 => unreachable!(),
+        };
+        let transcode_latin1 = self.transcoder(src, &dst, latin1);
+        let transcode_utf16 = self.transcoder(src, &dst, utf16);
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.instruction(Call(transcode_latin1.as_u32()));
+        self.instruction(LocalSet(dst.len.idx));
+        let src_len_tmp = self.local_set_new_tmp(src.opts.ptr());
+
+        // If the source was entirely consumed then the transcode completed
+        // and all that's necessary is to optionally shrink the buffer.
+        self.instruction(LocalGet(src_len_tmp.idx));
+        self.instruction(LocalGet(src.len.idx));
+        self.ptr_eq(src.opts);
+        self.instruction(If(BlockType::Empty)); // if latin1-or-utf16 block
+
+        // Test if the original byte length of the allocation is the same as
+        // the number of written bytes, and if not then shrink the buffer
+        // with a call to `realloc`.
+        self.instruction(LocalGet(dst_byte_len.idx));
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_ne(dst.opts);
+        self.instruction(If(BlockType::Empty));
+        self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
+        self.instruction(LocalGet(dst_byte_len.idx)); // old_size
+        self.ptr_uconst(dst.opts, 2); // align
+        self.instruction(LocalGet(dst.len.idx)); // new_size
+        self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+        self.instruction(End);
+
+        // In this block the latin1 encoding failed. The host transcode
+        // returned how many units were consumed from the source and how
+        // many bytes were written to the destination. Here the buffer is
+        // inflated and sized and the second utf16 intrinsic is invoked to
+        // perform the final inflation.
+        self.instruction(Else); // else latin1-or-utf16 block
+
+        // For utf8 validate that the inflated size is still within bounds.
+        if src_enc.width() == 1 {
+            self.validate_string_length_u8(src, 2);
+        }
+
+        // Reallocate the buffer with twice the source code units in byte
+        // size.
+        self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
+        self.instruction(LocalGet(dst_byte_len.idx)); // old_size
+        self.ptr_uconst(dst.opts, 2); // align
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
+        self.ptr_uconst(dst.opts, 1);
+        self.ptr_shl(dst.opts);
+        self.instruction(LocalTee(dst_byte_len.idx));
+        self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+
+        // Call the host utf16 transcoding function. This will inflate the
+        // prior latin1 bytes and then encode the rest of the source string
+        // as utf16 into the remaining space in the destination buffer.
+        self.instruction(LocalGet(src.ptr.idx));
+        self.instruction(LocalGet(src_len_tmp.idx));
+        if let FE::Utf16 = src_enc {
+            self.ptr_uconst(src.opts, 1);
+            self.ptr_shl(src.opts);
+        }
+        self.ptr_add(src.opts);
+        self.instruction(LocalGet(src.len.idx));
+        self.instruction(LocalGet(src_len_tmp.idx));
+        self.ptr_sub(src.opts);
+        self.instruction(LocalGet(dst.ptr.idx));
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
+        self.instruction(LocalGet(dst.len.idx));
+        self.instruction(Call(transcode_utf16.as_u32()));
+        self.instruction(LocalSet(dst.len.idx));
+
+        // If the returned number of code units written to the destination
+        // is not equal to the size of the allocation then the allocation is
+        // resized down to the appropriate size.
+        //
+        // Note that the byte size desired is `2*dst_len` and the current
+        // byte buffer size is `2*src_len` so the `2` factor isn't checked
+        // here, just the lengths.
+        self.instruction(LocalGet(dst.len.idx));
+        self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
+        self.ptr_ne(dst.opts);
+        self.instruction(If(BlockType::Empty));
+        self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
+        self.instruction(LocalGet(dst_byte_len.idx)); // old_size
+        self.ptr_uconst(dst.opts, 2); // align
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_uconst(dst.opts, 1);
+        self.ptr_shl(dst.opts);
+        self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
+        self.instruction(LocalSet(dst.ptr.idx));
+        self.instruction(End);
+
+        // Tag the returned pointer as utf16
+        self.instruction(LocalGet(dst.len.idx));
+        self.ptr_uconst(dst.opts, UTF16_TAG);
+        self.ptr_or(dst.opts);
+        self.instruction(LocalSet(dst.len.idx));
+
+        self.instruction(End); // end latin1-or-utf16 block
+
+        self.free_temp_local(src_len_tmp);
+        self.free_temp_local(dst_byte_len);
+
+        dst
+    }
+
+    fn validate_string_length(&mut self, src: &WasmString<'_>, dst: FE) {
+        self.validate_string_length_u8(src, dst.width())
+    }
+
+    fn validate_string_length_u8(&mut self, s: &WasmString<'_>, dst: u8) {
+        // Check to see if the source byte length is out of bounds in
+        // which case a trap is generated.
+        self.instruction(LocalGet(s.len.idx));
+        let max = MAX_STRING_BYTE_LENGTH / u32::from(dst);
+        self.ptr_uconst(s.opts, max);
+        self.ptr_ge_u(s.opts);
+        self.instruction(If(BlockType::Empty));
+        self.trap(Trap::StringLengthTooBig);
+        self.instruction(End);
+    }
+
+    fn transcoder(
+        &mut self,
+        src: &WasmString<'_>,
+        dst: &WasmString<'_>,
+        op: Transcode,
+    ) -> FuncIndex {
+        self.module.import_transcoder(Transcoder {
+            from_memory: src.opts.memory.unwrap(),
+            from_memory64: src.opts.memory64,
+            to_memory: dst.opts.memory.unwrap(),
+            to_memory64: dst.opts.memory64,
+            op,
+        })
+    }
+
+    fn validate_string_inbounds(&mut self, s: &WasmString<'_>, byte_len: u32) {
+        self.validate_memory_inbounds(s.opts, s.ptr.idx, byte_len, Trap::StringLengthOverflow)
+    }
+
+    fn validate_memory_inbounds(
+        &mut self,
+        opts: &Options,
+        ptr_local: u32,
+        byte_len_local: u32,
+        trap: Trap,
+    ) {
+        let extend_to_64 = |me: &mut Self| {
+            if !opts.memory64 {
+                me.instruction(I64ExtendI32U);
+            }
+        };
+
+        self.instruction(Block(BlockType::Empty));
+        self.instruction(Block(BlockType::Empty));
+
+        // Calculate the full byte size of memory with `memory.size`. Note that
+        // arithmetic here is done always in 64-bits to accomodate 4G memories.
+        // Additionally it's assumed that 64-bit memories never fill up
+        // entirely.
+        self.instruction(MemorySize(opts.memory.unwrap().as_u32()));
+        extend_to_64(self);
+        self.instruction(I64Const(16));
+        self.instruction(I64Shl);
+
+        // Calculate the end address of the string. This is done by adding the
+        // base pointer to the byte length. For 32-bit memories there's no need
+        // to check for overflow since everything is extended to 64-bit, but for
+        // 64-bit memories overflow is checked.
+        self.instruction(LocalGet(ptr_local));
+        extend_to_64(self);
+        self.instruction(LocalGet(byte_len_local));
+        extend_to_64(self);
+        self.instruction(I64Add);
+        if opts.memory64 {
+            let tmp = self.local_tee_new_tmp(ValType::I64);
+            self.instruction(LocalGet(ptr_local));
+            self.ptr_lt_u(opts);
+            self.instruction(BrIf(0));
+            self.instruction(LocalGet(tmp.idx));
+            self.free_temp_local(tmp);
+        }
+
+        // If the byte size of memory is greater than the final address of the
+        // string then the string is invalid. Note that if it's precisely equal
+        // then that's ok.
+        self.instruction(I64GeU);
+        self.instruction(BrIf(1));
+
+        self.instruction(End);
+        self.trap(trap);
+        self.instruction(End);
+    }
+
+    fn translate_list(
+        &mut self,
+        src_ty: TypeListIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_element_ty = &self.types[src_ty].element;
+        let dst_element_ty = match dst_ty {
+            InterfaceType::List(r) => &self.types[*r].element,
+            _ => panic!("expected a list"),
+        };
+        let src_opts = src.opts();
+        let dst_opts = dst.opts();
+        let (src_size, src_align) = self.types.size_align(src_opts, src_element_ty);
+        let (dst_size, dst_align) = self.types.size_align(dst_opts, dst_element_ty);
+
+        // Load the pointer/length of this list into temporary locals. These
+        // will be referenced a good deal so this just makes it easier to deal
+        // with them consistently below rather than trying to reload from memory
+        // for example.
+        match src {
+            Source::Stack(s) => {
+                assert_eq!(s.locals.len(), 2);
+                self.stack_get(&s.slice(0..1), src_opts.ptr());
+                self.stack_get(&s.slice(1..2), src_opts.ptr());
+            }
+            Source::Memory(mem) => {
+                self.ptr_load(mem);
+                self.ptr_load(&mem.bump(src_opts.ptr_size().into()));
+            }
+        }
+        let src_len = self.local_set_new_tmp(src_opts.ptr());
+        let src_ptr = self.local_set_new_tmp(src_opts.ptr());
+
+        // Create a `Memory` operand which will internally assert that the
+        // `src_ptr` value is properly aligned.
+        let src_mem = self.memory_operand(src_opts, src_ptr, src_align);
+
+        // Calculate the source/destination byte lengths into unique locals.
+        let src_byte_len = self.calculate_list_byte_len(src_opts, src_len.idx, src_size);
+        let dst_byte_len = if src_size == dst_size {
+            self.convert_src_len_to_dst(src_byte_len.idx, src_opts.ptr(), dst_opts.ptr());
+            self.local_set_new_tmp(dst_opts.ptr())
+        } else if src_opts.ptr() == dst_opts.ptr() {
+            self.calculate_list_byte_len(dst_opts, src_len.idx, dst_size)
+        } else {
+            self.convert_src_len_to_dst(src_byte_len.idx, src_opts.ptr(), dst_opts.ptr());
+            let tmp = self.local_set_new_tmp(dst_opts.ptr());
+            let ret = self.calculate_list_byte_len(dst_opts, tmp.idx, dst_size);
+            self.free_temp_local(tmp);
+            ret
+        };
+
+        // Here `realloc` is invoked (in a `malloc`-like fashion) to allocate
+        // space for the list in the destination memory. This will also
+        // internally insert checks that the returned pointer is aligned
+        // correctly for the destination.
+        let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), dst_align);
+
+        // With all the pointers and byte lengths verity that both the source
+        // and the destination buffers are in-bounds.
+        self.validate_memory_inbounds(
+            src_opts,
+            src_mem.addr.idx,
+            src_byte_len.idx,
+            Trap::ListByteLengthOverflow,
+        );
+        self.validate_memory_inbounds(
+            dst_opts,
+            dst_mem.addr.idx,
+            dst_byte_len.idx,
+            Trap::ListByteLengthOverflow,
+        );
+
+        self.free_temp_local(src_byte_len);
+        self.free_temp_local(dst_byte_len);
+
+        // This is the main body of the loop to actually translate list types.
+        // Note that if both element sizes are 0 then this won't actually do
+        // anything so the loop is removed entirely.
+        if src_size > 0 || dst_size > 0 {
+            // This block encompasses the entire loop and is use to exit before even
+            // entering the loop if the list size is zero.
+            self.instruction(Block(BlockType::Empty));
+
+            // Set the `remaining` local and only continue if it's > 0
+            self.instruction(LocalGet(src_len.idx));
+            let remaining = self.local_tee_new_tmp(src_opts.ptr());
+            self.ptr_eqz(src_opts);
+            self.instruction(BrIf(0));
+
+            // Initialize the two destination pointers to their initial values
+            self.instruction(LocalGet(src_mem.addr.idx));
+            let cur_src_ptr = self.local_set_new_tmp(src_opts.ptr());
+            self.instruction(LocalGet(dst_mem.addr.idx));
+            let cur_dst_ptr = self.local_set_new_tmp(dst_opts.ptr());
+
+            self.instruction(Loop(BlockType::Empty));
+
+            // Translate the next element in the list
+            let element_src = Source::Memory(Memory {
+                opts: src_opts,
+                offset: 0,
+                addr: TempLocal::new(cur_src_ptr.idx, cur_src_ptr.ty),
+            });
+            let element_dst = Destination::Memory(Memory {
+                opts: dst_opts,
+                offset: 0,
+                addr: TempLocal::new(cur_dst_ptr.idx, cur_dst_ptr.ty),
+            });
+            self.translate(src_element_ty, &element_src, dst_element_ty, &element_dst);
+
+            // Update the two loop pointers
+            if src_size > 0 {
+                self.instruction(LocalGet(cur_src_ptr.idx));
+                self.ptr_uconst(src_opts, src_size);
+                self.ptr_add(src_opts);
+                self.instruction(LocalSet(cur_src_ptr.idx));
+            }
+            if dst_size > 0 {
+                self.instruction(LocalGet(cur_dst_ptr.idx));
+                self.ptr_uconst(dst_opts, dst_size);
+                self.ptr_add(dst_opts);
+                self.instruction(LocalSet(cur_dst_ptr.idx));
+            }
+
+            // Update the remaining count, falling through to break out if it's zero
+            // now.
+            self.instruction(LocalGet(remaining.idx));
+            self.ptr_iconst(src_opts, -1);
+            self.ptr_add(src_opts);
+            self.instruction(LocalTee(remaining.idx));
+            self.ptr_br_if(src_opts, 0);
+            self.instruction(End); // end of loop
+            self.instruction(End); // end of block
+
+            self.free_temp_local(cur_dst_ptr);
+            self.free_temp_local(cur_src_ptr);
+            self.free_temp_local(remaining);
+        }
+
+        // Store the ptr/length in the desired destination
+        match dst {
+            Destination::Stack(s, _) => {
+                self.instruction(LocalGet(dst_mem.addr.idx));
+                self.stack_set(&s[..1], dst_opts.ptr());
+                self.convert_src_len_to_dst(src_len.idx, src_opts.ptr(), dst_opts.ptr());
+                self.stack_set(&s[1..], dst_opts.ptr());
+            }
+            Destination::Memory(mem) => {
+                self.instruction(LocalGet(mem.addr.idx));
+                self.instruction(LocalGet(dst_mem.addr.idx));
+                self.ptr_store(mem);
+                self.instruction(LocalGet(mem.addr.idx));
+                self.convert_src_len_to_dst(src_len.idx, src_opts.ptr(), dst_opts.ptr());
+                self.ptr_store(&mem.bump(dst_opts.ptr_size().into()));
+            }
+        }
+
+        self.free_temp_local(src_len);
+        self.free_temp_local(src_mem.addr);
+        self.free_temp_local(dst_mem.addr);
+    }
+
+    fn calculate_list_byte_len(
+        &mut self,
+        opts: &Options,
+        len_local: u32,
+        elt_size: u32,
+    ) -> TempLocal {
+        // Zero-size types are easy to handle here because the byte size of the
+        // destination is always zero.
+        if elt_size == 0 {
+            self.ptr_uconst(opts, 0);
+            return self.local_set_new_tmp(opts.ptr());
+        }
+
+        // For one-byte elements in the destination the check here can be a bit
+        // more optimal than the general case below. In these situations if the
+        // source pointer type is 32-bit then we're guaranteed to not overflow,
+        // so the source length is simply casted to the destination's type.
+        //
+        // If the source is 64-bit then all that needs to be checked is to
+        // ensure that it does not have the upper 32-bits set.
+        if elt_size == 1 {
+            if let ValType::I64 = opts.ptr() {
+                self.instruction(LocalGet(len_local));
+                self.instruction(I64Const(32));
+                self.instruction(I64ShrU);
+                self.instruction(I32WrapI64);
+                self.instruction(If(BlockType::Empty));
+                self.trap(Trap::ListByteLengthOverflow);
+                self.instruction(End);
+            }
+            self.instruction(LocalGet(len_local));
+            return self.local_set_new_tmp(opts.ptr());
+        }
+
+        // The main check implemented by this function is to verify that
+        // `src_len_local` does not exceed the 32-bit range. Byte sizes for
+        // lists must always fit in 32-bits to get transferred to 32-bit
+        // memories.
+        self.instruction(Block(BlockType::Empty));
+        self.instruction(Block(BlockType::Empty));
+        self.instruction(LocalGet(len_local));
+        match opts.ptr() {
+            // The source's list length is guaranteed to be less than 32-bits
+            // so simply extend it up to a 64-bit type for the multiplication
+            // below.
+            ValType::I32 => self.instruction(I64ExtendI32U),
+
+            // If the source is a 64-bit memory then if the item length doesn't
+            // fit in 32-bits the byte length definitely won't, so generate a
+            // branch to our overflow trap here if any of the upper 32-bits are set.
+            ValType::I64 => {
+                self.instruction(I64Const(32));
+                self.instruction(I64ShrU);
+                self.instruction(I32WrapI64);
+                self.instruction(BrIf(0));
+                self.instruction(LocalGet(len_local));
+            }
+
+            _ => unreachable!(),
+        }
+
+        // Next perform a 64-bit multiplication with the element byte size that
+        // is itself guaranteed to fit in 32-bits. The result is then checked
+        // to see if we overflowed the 32-bit space. The two input operands to
+        // the multiplication are guaranteed to be 32-bits at most which means
+        // that this multiplication shouldn't overflow.
+        //
+        // The result of the multiplication is saved into a local as well to
+        // get the result afterwards.
+        self.instruction(I64Const(elt_size.into()));
+        self.instruction(I64Mul);
+        let tmp = self.local_tee_new_tmp(ValType::I64);
+        // Branch to success if the upper 32-bits are zero, otherwise
+        // fall-through to the trap.
+        self.instruction(I64Const(32));
+        self.instruction(I64ShrU);
+        self.instruction(I64Eqz);
+        self.instruction(BrIf(1));
+        self.instruction(End);
+        self.trap(Trap::ListByteLengthOverflow);
+        self.instruction(End);
+
+        // If a fresh local was used to store the result of the multiplication
+        // then convert it down to 32-bits which should be guaranteed to not
+        // lose information at this point.
+        if opts.ptr() == ValType::I64 {
+            tmp
+        } else {
+            self.instruction(LocalGet(tmp.idx));
+            self.instruction(I32WrapI64);
+            self.free_temp_local(tmp);
+            self.local_set_new_tmp(ValType::I32)
+        }
+    }
+
+    fn convert_src_len_to_dst(
+        &mut self,
+        src_len_local: u32,
+        src_ptr_ty: ValType,
+        dst_ptr_ty: ValType,
+    ) {
+        self.instruction(LocalGet(src_len_local));
+        match (src_ptr_ty, dst_ptr_ty) {
+            (ValType::I32, ValType::I64) => self.instruction(I64ExtendI32U),
+            (ValType::I64, ValType::I32) => self.instruction(I32WrapI64),
+            (src, dst) => assert_eq!(src, dst),
+        }
+    }
+
+    fn translate_record(
+        &mut self,
+        src_ty: TypeRecordIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Record(r) => &self.types[*r],
+            _ => panic!("expected a record"),
+        };
+
+        // TODO: subtyping
+        assert_eq!(src_ty.fields.len(), dst_ty.fields.len());
+
+        // First a map is made of the source fields to where they're coming
+        // from (e.g. which offset or which locals). This map is keyed by the
+        // fields' names
+        let mut src_fields = HashMap::new();
+        for (i, src) in src
+            .record_field_srcs(self.types, src_ty.fields.iter().map(|f| f.ty))
+            .enumerate()
+        {
+            let field = &src_ty.fields[i];
+            src_fields.insert(&field.name, (src, &field.ty));
+        }
+
+        // .. and next translation is performed in the order of the destination
+        // fields in case the destination is the stack to ensure that the stack
+        // has the fields all in the right order.
+        //
+        // Note that the lookup in `src_fields` is an infallible lookup which
+        // will panic if the field isn't found.
+        //
+        // TODO: should that lookup be fallible with subtyping?
+        for (i, dst) in dst
+            .record_field_dsts(self.types, dst_ty.fields.iter().map(|f| f.ty))
+            .enumerate()
+        {
+            let field = &dst_ty.fields[i];
+            let (src, src_ty) = &src_fields[&field.name];
+            self.translate(src_ty, src, &field.ty, &dst);
+        }
+    }
+
+    fn translate_flags(
+        &mut self,
+        src_ty: TypeFlagsIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Flags(r) => &self.types[*r],
+            _ => panic!("expected a record"),
+        };
+
+        // TODO: subtyping
+        //
+        // Notably this implementation does not support reordering flags from
+        // the source to the destination nor having more flags in the
+        // destination. Currently this is a copy from source to destination
+        // in-bulk. Otherwise reordering indices would have to have some sort of
+        // fancy bit twiddling tricks or something like that.
+        assert_eq!(src_ty.names, dst_ty.names);
+        let cnt = src_ty.names.len();
+        match FlagsSize::from_count(cnt) {
+            FlagsSize::Size0 => {}
+            FlagsSize::Size1 => {
+                let mask = if cnt == 8 { 0xff } else { (1 << cnt) - 1 };
+                self.convert_u8_mask(src, dst, mask);
+            }
+            FlagsSize::Size2 => {
+                let mask = if cnt == 16 { 0xffff } else { (1 << cnt) - 1 };
+                self.convert_u16_mask(src, dst, mask);
+            }
+            FlagsSize::Size4Plus(n) => {
+                let srcs = src.record_field_srcs(self.types, (0..n).map(|_| InterfaceType::U32));
+                let dsts = dst.record_field_dsts(self.types, (0..n).map(|_| InterfaceType::U32));
+                let n = usize::from(n);
+                for (i, (src, dst)) in srcs.zip(dsts).enumerate() {
+                    let mask = if i == n - 1 && (cnt % 32 != 0) {
+                        (1 << (cnt % 32)) - 1
+                    } else {
+                        0xffffffff
+                    };
+                    self.convert_u32_mask(&src, &dst, mask);
+                }
+            }
+        }
+    }
+
+    fn translate_tuple(
+        &mut self,
+        src_ty: TypeTupleIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Tuple(t) => &self.types[*t],
+            _ => panic!("expected a tuple"),
+        };
+
+        // TODO: subtyping
+        assert_eq!(src_ty.types.len(), dst_ty.types.len());
+
+        let srcs = src
+            .record_field_srcs(self.types, src_ty.types.iter().copied())
+            .zip(src_ty.types.iter());
+        let dsts = dst
+            .record_field_dsts(self.types, dst_ty.types.iter().copied())
+            .zip(dst_ty.types.iter());
+        for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) {
+            self.translate(src_ty, &src, dst_ty, &dst);
+        }
+    }
+
+    fn translate_variant(
+        &mut self,
+        src_ty: TypeVariantIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Variant(t) => &self.types[*t],
+            _ => panic!("expected a variant"),
+        };
+
+        let src_info = variant_info(self.types, src_ty.cases.iter().map(|c| c.ty.as_ref()));
+        let dst_info = variant_info(self.types, dst_ty.cases.iter().map(|c| c.ty.as_ref()));
+
+        let iter = src_ty.cases.iter().enumerate().map(|(src_i, src_case)| {
+            let dst_i = dst_ty
+                .cases
+                .iter()
+                .position(|c| c.name == src_case.name)
+                .unwrap();
+            let dst_case = &dst_ty.cases[dst_i];
+            let src_i = u32::try_from(src_i).unwrap();
+            let dst_i = u32::try_from(dst_i).unwrap();
+            VariantCase {
+                src_i,
+                src_ty: src_case.ty.as_ref(),
+                dst_i,
+                dst_ty: dst_case.ty.as_ref(),
+            }
+        });
+        self.convert_variant(src, &src_info, dst, &dst_info, iter);
+    }
+
+    fn translate_union(
+        &mut self,
+        src_ty: TypeUnionIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Union(t) => &self.types[*t],
+            _ => panic!("expected an option"),
+        };
+        assert_eq!(src_ty.types.len(), dst_ty.types.len());
+        let src_info = variant_info(self.types, src_ty.types.iter().map(Some));
+        let dst_info = variant_info(self.types, dst_ty.types.iter().map(Some));
+
+        self.convert_variant(
+            src,
+            &src_info,
+            dst,
+            &dst_info,
+            src_ty
+                .types
+                .iter()
+                .zip(dst_ty.types.iter())
+                .enumerate()
+                .map(|(i, (src_ty, dst_ty))| {
+                    let i = u32::try_from(i).unwrap();
+                    VariantCase {
+                        src_i: i,
+                        dst_i: i,
+                        src_ty: Some(src_ty),
+                        dst_ty: Some(dst_ty),
+                    }
+                }),
+        );
+    }
+
+    fn translate_enum(
+        &mut self,
+        src_ty: TypeEnumIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Enum(t) => &self.types[*t],
+            _ => panic!("expected an option"),
+        };
+        let src_info = variant_info(self.types, src_ty.names.iter().map(|_| None));
+        let dst_info = variant_info(self.types, dst_ty.names.iter().map(|_| None));
+
+        self.convert_variant(
+            src,
+            &src_info,
+            dst,
+            &dst_info,
+            src_ty.names.iter().enumerate().map(|(src_i, src_name)| {
+                let dst_i = dst_ty.names.iter().position(|n| n == src_name).unwrap();
+                let src_i = u32::try_from(src_i).unwrap();
+                let dst_i = u32::try_from(dst_i).unwrap();
+                VariantCase {
+                    src_i,
+                    dst_i,
+                    src_ty: None,
+                    dst_ty: None,
+                }
+            }),
+        );
+    }
+
+    fn translate_option(
+        &mut self,
+        src_ty: TypeOptionIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty].ty;
+        let dst_ty = match dst_ty {
+            InterfaceType::Option(t) => &self.types[*t].ty,
+            _ => panic!("expected an option"),
+        };
+        let src_ty = Some(src_ty);
+        let dst_ty = Some(dst_ty);
+
+        let src_info = variant_info(self.types, [None, src_ty]);
+        let dst_info = variant_info(self.types, [None, dst_ty]);
+
+        self.convert_variant(
+            src,
+            &src_info,
+            dst,
+            &dst_info,
+            [
+                VariantCase {
+                    src_i: 0,
+                    dst_i: 0,
+                    src_ty: None,
+                    dst_ty: None,
+                },
+                VariantCase {
+                    src_i: 1,
+                    dst_i: 1,
+                    src_ty,
+                    dst_ty,
+                },
+            ]
+            .into_iter(),
+        );
+    }
+
+    fn translate_result(
+        &mut self,
+        src_ty: TypeResultIndex,
+        src: &Source<'_>,
+        dst_ty: &InterfaceType,
+        dst: &Destination,
+    ) {
+        let src_ty = &self.types[src_ty];
+        let dst_ty = match dst_ty {
+            InterfaceType::Result(t) => &self.types[*t],
+            _ => panic!("expected a result"),
+        };
+
+        let src_info = variant_info(self.types, [src_ty.ok.as_ref(), src_ty.err.as_ref()]);
+        let dst_info = variant_info(self.types, [dst_ty.ok.as_ref(), dst_ty.err.as_ref()]);
+
+        self.convert_variant(
+            src,
+            &src_info,
+            dst,
+            &dst_info,
+            [
+                VariantCase {
+                    src_i: 0,
+                    dst_i: 0,
+                    src_ty: src_ty.ok.as_ref(),
+                    dst_ty: dst_ty.ok.as_ref(),
+                },
+                VariantCase {
+                    src_i: 1,
+                    dst_i: 1,
+                    src_ty: src_ty.err.as_ref(),
+                    dst_ty: dst_ty.err.as_ref(),
+                },
+            ]
+            .into_iter(),
+        );
+    }
+
+    fn convert_variant<'a>(
+        &mut self,
+        src: &Source<'_>,
+        src_info: &VariantInfo,
+        dst: &Destination,
+        dst_info: &VariantInfo,
+        src_cases: impl ExactSizeIterator<Item = VariantCase<'a>>,
+    ) {
+        // The outermost block is special since it has the result type of the
+        // translation here. That will depend on the `dst`.
+        let outer_block_ty = match dst {
+            Destination::Stack(dst_flat, _) => match dst_flat.len() {
+                0 => BlockType::Empty,
+                1 => BlockType::Result(dst_flat[0]),
+                _ => {
+                    let ty = self.module.core_types.function(&[], &dst_flat);
+                    BlockType::FunctionType(ty)
+                }
+            },
+            Destination::Memory(_) => BlockType::Empty,
+        };
+        self.instruction(Block(outer_block_ty));
+
+        // After the outermost block generate a new block for each of the
+        // remaining cases.
+        let src_cases_len = src_cases.len();
+        for _ in 0..src_cases_len - 1 {
+            self.instruction(Block(BlockType::Empty));
+        }
+
+        // Generate a block for an invalid variant discriminant
+        self.instruction(Block(BlockType::Empty));
+
+        // And generate one final block that we'll be jumping out of with the
+        // `br_table`
+        self.instruction(Block(BlockType::Empty));
+
+        // Load the discriminant
+        match src {
+            Source::Stack(s) => self.stack_get(&s.slice(0..1), ValType::I32),
+            Source::Memory(mem) => match src_info.size {
+                DiscriminantSize::Size1 => self.i32_load8u(mem),
+                DiscriminantSize::Size2 => self.i32_load16u(mem),
+                DiscriminantSize::Size4 => self.i32_load(mem),
+            },
+        }
+
+        // Generate the `br_table` for the discriminant. Each case has an
+        // offset of 1 to skip the trapping block.
+        let mut targets = Vec::new();
+        for i in 0..src_cases_len {
+            targets.push((i + 1) as u32);
+        }
+        self.instruction(BrTable(targets[..].into(), 0));
+        self.instruction(End); // end the `br_table` block
+
+        self.trap(Trap::InvalidDiscriminant);
+        self.instruction(End); // end the "invalid discriminant" block
+
+        // Translate each case individually within its own block. Note that the
+        // iteration order here places the first case in the innermost block
+        // and the last case in the outermost block. This matches the order
+        // of the jump targets in the `br_table` instruction.
+        let src_cases_len = u32::try_from(src_cases_len).unwrap();
+        for case in src_cases {
+            let VariantCase {
+                src_i,
+                src_ty,
+                dst_i,
+                dst_ty,
+            } = case;
+
+            // Translate the discriminant here, noting that `dst_i` may be
+            // different than `src_i`.
+            self.push_dst_addr(dst);
+            self.instruction(I32Const(dst_i as i32));
+            match dst {
+                Destination::Stack(stack, _) => self.stack_set(&stack[..1], ValType::I32),
+                Destination::Memory(mem) => match dst_info.size {
+                    DiscriminantSize::Size1 => self.i32_store8(mem),
+                    DiscriminantSize::Size2 => self.i32_store16(mem),
+                    DiscriminantSize::Size4 => self.i32_store(mem),
+                },
+            }
+
+            let src_payload = src.payload_src(self.types, src_info, src_ty);
+            let dst_payload = dst.payload_dst(self.types, dst_info, dst_ty);
+
+            // Translate the payload of this case using the various types from
+            // the dst/src.
+            match (src_ty, dst_ty) {
+                (Some(src_ty), Some(dst_ty)) => {
+                    self.translate(src_ty, &src_payload, dst_ty, &dst_payload);
+                }
+                (None, None) => {}
+                _ => unimplemented!(),
+            }
+
+            // If the results of this translation were placed on the stack then
+            // the stack values may need to be padded with more zeros due to
+            // this particular case being possibly smaller than the entire
+            // variant. That's handled here by pushing remaining zeros after
+            // accounting for the discriminant pushed as well as the results of
+            // this individual payload.
+            if let Destination::Stack(payload_results, _) = dst_payload {
+                if let Destination::Stack(dst_results, _) = dst {
+                    let remaining = &dst_results[1..][payload_results.len()..];
+                    for ty in remaining {
+                        match ty {
+                            ValType::I32 => self.instruction(I32Const(0)),
+                            ValType::I64 => self.instruction(I64Const(0)),
+                            ValType::F32 => self.instruction(F32Const(0.0)),
+                            ValType::F64 => self.instruction(F64Const(0.0)),
+                            _ => unreachable!(),
+                        }
+                    }
+                }
+            }
+
+            // Branch to the outermost block. Note that this isn't needed for
+            // the outermost case since it simply falls through.
+            if src_i != src_cases_len - 1 {
+                self.instruction(Br(src_cases_len - src_i - 1));
+            }
+            self.instruction(End); // end this case's block
+        }
+    }
+
+    fn trap_if_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, trap: Trap) {
+        self.instruction(GlobalGet(flags_global.as_u32()));
+        self.instruction(I32Const(flag_to_test));
+        self.instruction(I32And);
+        self.instruction(I32Eqz);
+        self.instruction(If(BlockType::Empty));
+        self.trap(trap);
+        self.instruction(End);
+    }
+
+    fn assert_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, msg: &'static str) {
+        self.instruction(GlobalGet(flags_global.as_u32()));
+        self.instruction(I32Const(flag_to_test));
+        self.instruction(I32And);
+        self.instruction(If(BlockType::Empty));
+        self.trap(Trap::AssertFailed(msg));
+        self.instruction(End);
+    }
+
+    fn set_flag(&mut self, flags_global: GlobalIndex, flag_to_set: i32, value: bool) {
+        self.instruction(GlobalGet(flags_global.as_u32()));
+        if value {
+            self.instruction(I32Const(flag_to_set));
+            self.instruction(I32Or);
+        } else {
+            self.instruction(I32Const(!flag_to_set));
+            self.instruction(I32And);
+        }
+        self.instruction(GlobalSet(flags_global.as_u32()));
+    }
+
+    fn verify_aligned(&mut self, opts: &Options, addr_local: u32, align: u32) {
+        // If the alignment is 1 then everything is trivially aligned and the
+        // check can be omitted.
+        if align == 1 {
+            return;
+        }
+        self.instruction(LocalGet(addr_local));
+        assert!(align.is_power_of_two());
+        self.ptr_uconst(opts, align - 1);
+        self.ptr_and(opts);
+        self.ptr_if(opts, BlockType::Empty);
+        self.trap(Trap::UnalignedPointer);
+        self.instruction(End);
+    }
+
+    fn assert_aligned(&mut self, ty: &InterfaceType, mem: &Memory) {
+        if !self.module.debug {
+            return;
+        }
+        let align = self.types.align(mem.opts, ty);
+        if align == 1 {
+            return;
+        }
+        assert!(align.is_power_of_two());
+        self.instruction(LocalGet(mem.addr.idx));
+        self.ptr_uconst(mem.opts, mem.offset);
+        self.ptr_add(mem.opts);
+        self.ptr_uconst(mem.opts, align - 1);
+        self.ptr_and(mem.opts);
+        self.ptr_if(mem.opts, BlockType::Empty);
+        self.trap(Trap::AssertFailed("pointer not aligned"));
+        self.instruction(End);
+    }
+
+    fn malloc<'a>(&mut self, opts: &'a Options, size: MallocSize, align: u32) -> Memory<'a> {
+        let realloc = opts.realloc.unwrap();
+        self.ptr_uconst(opts, 0);
+        self.ptr_uconst(opts, 0);
+        self.ptr_uconst(opts, align);
+        match size {
+            MallocSize::Const(size) => self.ptr_uconst(opts, size),
+            MallocSize::Local(idx) => self.instruction(LocalGet(idx)),
+        }
+        self.instruction(Call(realloc.as_u32()));
+        let addr = self.local_set_new_tmp(opts.ptr());
+        self.memory_operand(opts, addr, align)
+    }
+
+    fn memory_operand<'a>(&mut self, opts: &'a Options, addr: TempLocal, align: u32) -> Memory<'a> {
+        let ret = Memory {
+            addr,
+            offset: 0,
+            opts,
+        };
+        self.verify_aligned(opts, ret.addr.idx, align);
+        ret
+    }
+
+    /// Generates a new local in this function of the `ty` specified,
+    /// initializing it with the top value on the current wasm stack.
+    ///
+    /// The returned `TempLocal` must be freed after it is finished with
+    /// `free_temp_local`.
+    fn local_tee_new_tmp(&mut self, ty: ValType) -> TempLocal {
+        self.gen_temp_local(ty, LocalTee)
+    }
+
+    /// Same as `local_tee_new_tmp` but initializes the local with `LocalSet`
+    /// instead of `LocalTee`.
+    fn local_set_new_tmp(&mut self, ty: ValType) -> TempLocal {
+        self.gen_temp_local(ty, LocalSet)
+    }
+
+    fn gen_temp_local(&mut self, ty: ValType, insn: fn(u32) -> Instruction<'static>) -> TempLocal {
+        // First check to see if any locals are available in this function which
+        // were previously generated but are no longer in use.
+        if let Some(idx) = self.free_locals.get_mut(&ty).and_then(|v| v.pop()) {
+            self.instruction(insn(idx));
+            return TempLocal {
+                ty,
+                idx,
+                needs_free: true,
+            };
+        }
+
+        // Failing that generate a fresh new local.
+        let locals = &mut self.module.funcs[self.result].locals;
+        match locals.last_mut() {
+            Some((cnt, prev_ty)) if ty == *prev_ty => *cnt += 1,
+            _ => locals.push((1, ty)),
+        }
+        self.nlocals += 1;
+        let idx = self.nlocals - 1;
+        self.instruction(insn(idx));
+        TempLocal {
+            ty,
+            idx,
+            needs_free: true,
+        }
+    }
+
+    /// Used to release a `TempLocal` from a particular lexical scope to allow
+    /// its possible reuse in later scopes.
+    fn free_temp_local(&mut self, mut local: TempLocal) {
+        assert!(local.needs_free);
+        self.free_locals
+            .entry(local.ty)
+            .or_insert(Vec::new())
+            .push(local.idx);
+        local.needs_free = false;
+    }
+
+    fn instruction(&mut self, instr: Instruction) {
+        instr.encode(&mut self.code);
+    }
+
+    fn trap(&mut self, trap: Trap) {
+        self.traps.push((self.code.len(), trap));
+        self.instruction(Unreachable);
+    }
+
+    /// Flushes out the current `code` instructions (and `traps` if there are
+    /// any) into the destination function.
+    ///
+    /// This is a noop if no instructions have been encoded yet.
+    fn flush_code(&mut self) {
+        if self.code.is_empty() {
+            return;
+        }
+        self.module.funcs[self.result].body.push(Body::Raw(
+            mem::take(&mut self.code),
+            mem::take(&mut self.traps),
+        ));
+    }
+
+    fn finish(mut self) {
+        // Append the final `end` instruction which all functions require, and
+        // then empty out the temporary buffer in `Compiler`.
+        self.instruction(End);
+        self.flush_code();
+
+        // Flag the function as "done" which helps with an assert later on in
+        // emission that everything was eventually finished.
+        self.module.funcs[self.result].filled_in = true;
+    }
+
+    /// Fetches the value contained with the local specified by `stack` and
+    /// converts it to `dst_ty`.
+    ///
+    /// This is only intended for use in primitive operations where `stack` is
+    /// guaranteed to have only one local. The type of the local on the stack is
+    /// then converted to `dst_ty` appropriately. Note that the types may be
+    /// different due to the "flattening" of variant types.
+    fn stack_get(&mut self, stack: &Stack<'_>, dst_ty: ValType) {
+        assert_eq!(stack.locals.len(), 1);
+        let (idx, src_ty) = stack.locals[0];
+        self.instruction(LocalGet(idx));
+        match (src_ty, dst_ty) {
+            (ValType::I32, ValType::I32)
+            | (ValType::I64, ValType::I64)
+            | (ValType::F32, ValType::F32)
+            | (ValType::F64, ValType::F64) => {}
+
+            (ValType::I32, ValType::F32) => self.instruction(F32ReinterpretI32),
+            (ValType::I64, ValType::I32) => {
+                self.assert_i64_upper_bits_not_set(idx);
+                self.instruction(I32WrapI64);
+            }
+            (ValType::I64, ValType::F64) => self.instruction(F64ReinterpretI64),
+            (ValType::I64, ValType::F32) => {
+                self.assert_i64_upper_bits_not_set(idx);
+                self.instruction(I32WrapI64);
+                self.instruction(F32ReinterpretI32);
+            }
+
+            // should not be possible given the `join` function for variants
+            (ValType::I32, ValType::I64)
+            | (ValType::I32, ValType::F64)
+            | (ValType::F32, ValType::I32)
+            | (ValType::F32, ValType::I64)
+            | (ValType::F32, ValType::F64)
+            | (ValType::F64, ValType::I32)
+            | (ValType::F64, ValType::I64)
+            | (ValType::F64, ValType::F32)
+
+            // not used in the component model
+            | (ValType::Ref(_), _)
+            | (_, ValType::Ref(_))
+            | (ValType::V128, _)
+            | (_, ValType::V128) => {
+                panic!("cannot get {dst_ty:?} from {src_ty:?} local");
+            }
+        }
+    }
+
+    fn assert_i64_upper_bits_not_set(&mut self, local: u32) {
+        if !self.module.debug {
+            return;
+        }
+        self.instruction(LocalGet(local));
+        self.instruction(I64Const(32));
+        self.instruction(I64ShrU);
+        self.instruction(I32WrapI64);
+        self.instruction(If(BlockType::Empty));
+        self.trap(Trap::AssertFailed("upper bits are unexpectedly set"));
+        self.instruction(End);
+    }
+
+    /// Converts the top value on the WebAssembly stack which has type
+    /// `src_ty` to `dst_tys[0]`.
+    ///
+    /// This is only intended for conversion of primitives where the `dst_tys`
+    /// list is known to be of length 1.
+    fn stack_set(&mut self, dst_tys: &[ValType], src_ty: ValType) {
+        assert_eq!(dst_tys.len(), 1);
+        let dst_ty = dst_tys[0];
+        match (src_ty, dst_ty) {
+            (ValType::I32, ValType::I32)
+            | (ValType::I64, ValType::I64)
+            | (ValType::F32, ValType::F32)
+            | (ValType::F64, ValType::F64) => {}
+
+            (ValType::F32, ValType::I32) => self.instruction(I32ReinterpretF32),
+            (ValType::I32, ValType::I64) => self.instruction(I64ExtendI32U),
+            (ValType::F64, ValType::I64) => self.instruction(I64ReinterpretF64),
+            (ValType::F32, ValType::I64) => {
+                self.instruction(I32ReinterpretF32);
+                self.instruction(I64ExtendI32U);
+            }
+
+            // should not be possible given the `join` function for variants
+            (ValType::I64, ValType::I32)
+            | (ValType::F64, ValType::I32)
+            | (ValType::I32, ValType::F32)
+            | (ValType::I64, ValType::F32)
+            | (ValType::F64, ValType::F32)
+            | (ValType::I32, ValType::F64)
+            | (ValType::I64, ValType::F64)
+            | (ValType::F32, ValType::F64)
+
+            // not used in the component model
+            | (ValType::Ref(_), _)
+            | (_, ValType::Ref(_))
+            | (ValType::V128, _)
+            | (_, ValType::V128) => {
+                panic!("cannot get {dst_ty:?} from {src_ty:?} local");
+            }
+        }
+    }
+
+    fn i32_load8u(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(I32Load8U(mem.memarg(0)));
+    }
+
+    fn i32_load8s(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(I32Load8S(mem.memarg(0)));
+    }
+
+    fn i32_load16u(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(I32Load16U(mem.memarg(1)));
+    }
+
+    fn i32_load16s(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(I32Load16S(mem.memarg(1)));
+    }
+
+    fn i32_load(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(I32Load(mem.memarg(2)));
+    }
+
+    fn i64_load(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(I64Load(mem.memarg(3)));
+    }
+
+    fn ptr_load(&mut self, mem: &Memory) {
+        if mem.opts.memory64 {
+            self.i64_load(mem);
+        } else {
+            self.i32_load(mem);
+        }
+    }
+
+    fn ptr_add(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Add);
+        } else {
+            self.instruction(I32Add);
+        }
+    }
+
+    fn ptr_sub(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Sub);
+        } else {
+            self.instruction(I32Sub);
+        }
+    }
+
+    fn ptr_mul(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Mul);
+        } else {
+            self.instruction(I32Mul);
+        }
+    }
+
+    fn ptr_ge_u(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64GeU);
+        } else {
+            self.instruction(I32GeU);
+        }
+    }
+
+    fn ptr_lt_u(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64LtU);
+        } else {
+            self.instruction(I32LtU);
+        }
+    }
+
+    fn ptr_shl(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Shl);
+        } else {
+            self.instruction(I32Shl);
+        }
+    }
+
+    fn ptr_eqz(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Eqz);
+        } else {
+            self.instruction(I32Eqz);
+        }
+    }
+
+    fn ptr_uconst(&mut self, opts: &Options, val: u32) {
+        if opts.memory64 {
+            self.instruction(I64Const(val.into()));
+        } else {
+            self.instruction(I32Const(val as i32));
+        }
+    }
+
+    fn ptr_iconst(&mut self, opts: &Options, val: i32) {
+        if opts.memory64 {
+            self.instruction(I64Const(val.into()));
+        } else {
+            self.instruction(I32Const(val));
+        }
+    }
+
+    fn ptr_eq(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Eq);
+        } else {
+            self.instruction(I32Eq);
+        }
+    }
+
+    fn ptr_ne(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Ne);
+        } else {
+            self.instruction(I32Ne);
+        }
+    }
+
+    fn ptr_and(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64And);
+        } else {
+            self.instruction(I32And);
+        }
+    }
+
+    fn ptr_or(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Or);
+        } else {
+            self.instruction(I32Or);
+        }
+    }
+
+    fn ptr_xor(&mut self, opts: &Options) {
+        if opts.memory64 {
+            self.instruction(I64Xor);
+        } else {
+            self.instruction(I32Xor);
+        }
+    }
+
+    fn ptr_if(&mut self, opts: &Options, ty: BlockType) {
+        if opts.memory64 {
+            self.instruction(I64Const(0));
+            self.instruction(I64Ne);
+        }
+        self.instruction(If(ty));
+    }
+
+    fn ptr_br_if(&mut self, opts: &Options, depth: u32) {
+        if opts.memory64 {
+            self.instruction(I64Const(0));
+            self.instruction(I64Ne);
+        }
+        self.instruction(BrIf(depth));
+    }
+
+    fn f32_load(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(F32Load(mem.memarg(2)));
+    }
+
+    fn f64_load(&mut self, mem: &Memory) {
+        self.instruction(LocalGet(mem.addr.idx));
+        self.instruction(F64Load(mem.memarg(3)));
+    }
+
+    fn push_dst_addr(&mut self, dst: &Destination) {
+        if let Destination::Memory(mem) = dst {
+            self.instruction(LocalGet(mem.addr.idx));
+        }
+    }
+
+    fn i32_store8(&mut self, mem: &Memory) {
+        self.instruction(I32Store8(mem.memarg(0)));
+    }
+
+    fn i32_store16(&mut self, mem: &Memory) {
+        self.instruction(I32Store16(mem.memarg(1)));
+    }
+
+    fn i32_store(&mut self, mem: &Memory) {
+        self.instruction(I32Store(mem.memarg(2)));
+    }
+
+    fn i64_store(&mut self, mem: &Memory) {
+        self.instruction(I64Store(mem.memarg(3)));
+    }
+
+    fn ptr_store(&mut self, mem: &Memory) {
+        if mem.opts.memory64 {
+            self.i64_store(mem);
+        } else {
+            self.i32_store(mem);
+        }
+    }
+
+    fn f32_store(&mut self, mem: &Memory) {
+        self.instruction(F32Store(mem.memarg(2)));
+    }
+
+    fn f64_store(&mut self, mem: &Memory) {
+        self.instruction(F64Store(mem.memarg(3)));
+    }
+}
+
+impl<'a> Source<'a> {
+    /// Given this `Source` returns an iterator over the `Source` for each of
+    /// the component `fields` specified.
+    ///
+    /// This will automatically slice stack-based locals to the appropriate
+    /// width for each component type and additionally calculate the appropriate
+    /// offset for each memory-based type.
+    fn record_field_srcs<'b>(
+        &'b self,
+        types: &'b ComponentTypesBuilder,
+        fields: impl IntoIterator<Item = InterfaceType> + 'b,
+    ) -> impl Iterator<Item = Source<'a>> + 'b
+    where
+        'a: 'b,
+    {
+        let mut offset = 0;
+        fields.into_iter().map(move |ty| match self {
+            Source::Memory(mem) => {
+                let mem = next_field_offset(&mut offset, types, &ty, mem);
+                Source::Memory(mem)
+            }
+            Source::Stack(stack) => {
+                let cnt = types.flat_types(&ty).unwrap().len() as u32;
+                offset += cnt;
+                Source::Stack(stack.slice((offset - cnt) as usize..offset as usize))
+            }
+        })
+    }
+
+    /// Returns the corresponding discriminant source and payload source f
+    fn payload_src(
+        &self,
+        types: &ComponentTypesBuilder,
+        info: &VariantInfo,
+        case: Option<&InterfaceType>,
+    ) -> Source<'a> {
+        match self {
+            Source::Stack(s) => {
+                let flat_len = match case {
+                    Some(case) => types.flat_types(case).unwrap().len(),
+                    None => 0,
+                };
+                Source::Stack(s.slice(1..s.locals.len()).slice(0..flat_len))
+            }
+            Source::Memory(mem) => {
+                let mem = if mem.opts.memory64 {
+                    mem.bump(info.payload_offset64)
+                } else {
+                    mem.bump(info.payload_offset32)
+                };
+                Source::Memory(mem)
+            }
+        }
+    }
+
+    fn opts(&self) -> &'a Options {
+        match self {
+            Source::Stack(s) => s.opts,
+            Source::Memory(mem) => mem.opts,
+        }
+    }
+}
+
+impl<'a> Destination<'a> {
+    /// Same as `Source::record_field_srcs` but for destinations.
+    fn record_field_dsts<'b>(
+        &'b self,
+        types: &'b ComponentTypesBuilder,
+        fields: impl IntoIterator<Item = InterfaceType> + 'b,
+    ) -> impl Iterator<Item = Destination> + 'b
+    where
+        'a: 'b,
+    {
+        let mut offset = 0;
+        fields.into_iter().map(move |ty| match self {
+            Destination::Memory(mem) => {
+                let mem = next_field_offset(&mut offset, types, &ty, mem);
+                Destination::Memory(mem)
+            }
+            Destination::Stack(s, opts) => {
+                let cnt = types.flat_types(&ty).unwrap().len() as u32;
+                offset += cnt;
+                Destination::Stack(&s[(offset - cnt) as usize..offset as usize], opts)
+            }
+        })
+    }
+
+    /// Returns the corresponding discriminant source and payload source f
+    fn payload_dst(
+        &self,
+        types: &ComponentTypesBuilder,
+        info: &VariantInfo,
+        case: Option<&InterfaceType>,
+    ) -> Destination {
+        match self {
+            Destination::Stack(s, opts) => {
+                let flat_len = match case {
+                    Some(case) => types.flat_types(case).unwrap().len(),
+                    None => 0,
+                };
+                Destination::Stack(&s[1..][..flat_len], opts)
+            }
+            Destination::Memory(mem) => {
+                let mem = if mem.opts.memory64 {
+                    mem.bump(info.payload_offset64)
+                } else {
+                    mem.bump(info.payload_offset32)
+                };
+                Destination::Memory(mem)
+            }
+        }
+    }
+
+    fn opts(&self) -> &'a Options {
+        match self {
+            Destination::Stack(_, opts) => opts,
+            Destination::Memory(mem) => mem.opts,
+        }
+    }
+}
+
+fn next_field_offset<'a>(
+    offset: &mut u32,
+    types: &ComponentTypesBuilder,
+    field: &InterfaceType,
+    mem: &Memory<'a>,
+) -> Memory<'a> {
+    let abi = types.canonical_abi(field);
+    let offset = if mem.opts.memory64 {
+        abi.next_field64(offset)
+    } else {
+        abi.next_field32(offset)
+    };
+    mem.bump(offset)
+}
+
+impl<'a> Memory<'a> {
+    fn memarg(&self, align: u32) -> MemArg {
+        MemArg {
+            offset: u64::from(self.offset),
+            align,
+            memory_index: self.opts.memory.unwrap().as_u32(),
+        }
+    }
+
+    fn bump(&self, offset: u32) -> Memory<'a> {
+        Memory {
+            opts: self.opts,
+            addr: TempLocal::new(self.addr.idx, self.addr.ty),
+            offset: self.offset + offset,
+        }
+    }
+}
+
+impl<'a> Stack<'a> {
+    fn slice(&self, range: Range<usize>) -> Stack<'a> {
+        Stack {
+            locals: &self.locals[range],
+            opts: self.opts,
+        }
+    }
+}
+
+struct VariantCase<'a> {
+    src_i: u32,
+    src_ty: Option<&'a InterfaceType>,
+    dst_i: u32,
+    dst_ty: Option<&'a InterfaceType>,
+}
+
+fn variant_info<'a, I>(types: &ComponentTypesBuilder, cases: I) -> VariantInfo
+where
+    I: IntoIterator<Item = Option<&'a InterfaceType>>,
+    I::IntoIter: ExactSizeIterator,
+{
+    VariantInfo::new(
+        cases
+            .into_iter()
+            .map(|ty| ty.map(|ty| types.canonical_abi(ty))),
+    )
+    .0
+}
+
+enum MallocSize {
+    Const(u32),
+    Local(u32),
+}
+
+struct WasmString<'a> {
+    ptr: TempLocal,
+    len: TempLocal,
+    opts: &'a Options,
+}
+
+struct TempLocal {
+    idx: u32,
+    ty: ValType,
+    needs_free: bool,
+}
+
+impl TempLocal {
+    fn new(idx: u32, ty: ValType) -> TempLocal {
+        TempLocal {
+            idx,
+            ty,
+            needs_free: false,
+        }
+    }
+}
+
+impl std::ops::Drop for TempLocal {
+    fn drop(&mut self) {
+        if self.needs_free {
+            panic!("temporary local not free'd");
+        }
+    }
+}
+
+impl From<FlatType> for ValType {
+    fn from(ty: FlatType) -> ValType {
+        match ty {
+            FlatType::I32 => ValType::I32,
+            FlatType::I64 => ValType::I64,
+            FlatType::F32 => ValType::F32,
+            FlatType::F64 => ValType::F64,
+        }
+    }
+}