wasmtime 36.0.0 → 36.0.1

data/ext/cargo-vendor/winch-codegen-36.0.19/src/isa/x64/masm.rs

@@ -0,0 +1,3603 @@
+use super::{
+    RegAlloc,
+    abi::X64ABI,
+    address::Address,
+    asm::{Assembler, PatchableAddToReg, VcmpKind, VcvtKind, VroundMode},
+    regs::{self, rbp, rsp, scratch_fpr_bitset, scratch_gpr_bitset},
+};
+use anyhow::{Result, anyhow, bail};
+
+use crate::masm::{
+    DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm, Imm as I,
+    IntCmpKind, IntScratch, LaneSelector, LoadKind, MacroAssembler as Masm, MulWideKind,
+    OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, Scratch, ScratchType,
+    ShiftKind, SplatKind, StoreKind, TRUSTED_FLAGS, TrapCode, TruncKind, UNTRUSTED_FLAGS,
+    V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind,
+    V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind, V128SubKind, V128TruncKind,
+    VectorCompareKind, VectorEqualityKind, Zero,
+};
+use crate::{
+    abi::{self, LocalSlot, align_to, calculate_frame_adjustment},
+    codegen::{CodeGenContext, CodeGenError, Emission, FuncEnv, ptr_type_from_ptr_size},
+    stack::{TypedReg, Val},
+};
+use crate::{
+    abi::{ABI, vmctx},
+    masm::{SPOffset, StackSlot},
+};
+use crate::{
+    isa::{
+        CallingConvention,
+        reg::{Reg, RegClass, WritableReg, writable},
+    },
+    masm::CalleeKind,
+};
+use cranelift_codegen::{
+    Final, MachBufferFinalized, MachLabel,
+    binemit::CodeOffset,
+    ir::{MemFlags, RelSourceLoc, SourceLoc},
+    isa::{
+        unwind::UnwindInst,
+        x64::{AtomicRmwSeqOp, args::CC, settings as x64_settings},
+    },
+    settings,
+};
+use wasmtime_cranelift::TRAP_UNREACHABLE;
+use wasmtime_environ::{PtrSize, WasmValType};
+
+// Taken from `cranelift/codegen/src/isa/x64/lower/isle.rs`
+// Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we
+// need to fix up the bits that migrate from one half of the lane to the
+// other. Each 16-byte mask is indexed by the shift amount: e.g. if we shift
+// right by 0 (no movement), we want to retain all the bits so we mask with
+// `0xff`; if we shift right by 1, we want to retain all bits except the MSB so
+// we mask with `0x7f`; etc.
+
+#[rustfmt::skip] // Preserve 16 bytes (i.e. one mask) per row.
+const I8X16_ISHL_MASKS: [u8; 128] = [
+    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe,
+    0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc,
+    0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
+    0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0,
+    0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0,
+    0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0,
+    0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+];
+
+#[rustfmt::skip] // Preserve 16 bytes (i.e. one mask) per row.
+const I8X16_USHR_MASKS: [u8; 128] = [
+    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+    0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+    0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
+    0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+    0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+];
+
+/// x64 MacroAssembler.
+pub(crate) struct MacroAssembler {
+    /// Stack pointer offset.
+    sp_offset: u32,
+    /// This value represents the maximum stack size seen while compiling the function. While the
+    /// function is still being compiled its value will not be valid (the stack will grow and
+    /// shrink as space is reserved and freed during compilation), but once all instructions have
+    /// been seen this value will be the maximum stack usage seen.
+    sp_max: u32,
+    /// Add instructions that are used to add the constant stack max to a register.
+    stack_max_use_add: Option<PatchableAddToReg>,
+    /// Low level assembler.
+    asm: Assembler,
+    /// ISA flags.
+    flags: x64_settings::Flags,
+    /// Shared flags.vmcontext_store_context
+    shared_flags: settings::Flags,
+    /// The target pointer size.
+    ptr_size: OperandSize,
+    /// Scratch register scope.
+    scratch_scope: RegAlloc,
+}
+
+impl Masm for MacroAssembler {
+    type Address = Address;
+    type Ptr = u8;
+    type ABI = X64ABI;
+
+    fn frame_setup(&mut self) -> Result<()> {
+        let frame_pointer = rbp();
+        let stack_pointer = rsp();
+
+        self.asm.push_r(frame_pointer);
+
+        if self.shared_flags.unwind_info() {
+            self.asm.unwind_inst(UnwindInst::PushFrameRegs {
+                offset_upward_to_caller_sp: Self::ABI::arg_base_offset().into(),
+            })
+        }
+
+        self.asm
+            .mov_rr(stack_pointer, writable!(frame_pointer), OperandSize::S64);
+
+        Ok(())
+    }
+
+    fn check_stack(&mut self, vmctx: Reg) -> Result<()> {
+        let ptr_size: u8 = self.ptr_size.bytes().try_into().unwrap();
+
+        self.with_scratch::<IntScratch, _>(|masm, scratch| {
+            masm.load_ptr(
+                masm.address_at_reg(vmctx, ptr_size.vmcontext_store_context().into())?,
+                scratch.writable(),
+            )?;
+
+            masm.load_ptr(
+                Address::offset(
+                    scratch.inner(),
+                    ptr_size.vmstore_context_stack_limit().into(),
+                ),
+                scratch.writable(),
+            )?;
+
+            masm.add_stack_max(scratch.inner());
+
+            masm.asm.cmp_rr(scratch.inner(), regs::rsp(), masm.ptr_size);
+            masm.asm.trapif(IntCmpKind::GtU, TrapCode::STACK_OVERFLOW);
+            anyhow::Ok(())
+        })?;
+
+        // Emit unwind info.
+        if self.shared_flags.unwind_info() {
+            self.asm.unwind_inst(UnwindInst::DefineNewFrame {
+                offset_upward_to_caller_sp: Self::ABI::arg_base_offset().into(),
+
+                // The Winch calling convention has no callee-save registers, so nothing will be
+                // clobbered.
+                offset_downward_to_clobbers: 0,
+            })
+        }
+        Ok(())
+    }
+
+    fn push(&mut self, reg: Reg, size: OperandSize) -> Result<StackSlot> {
+        let bytes = match (reg.class(), size) {
+            (RegClass::Int, OperandSize::S64) => {
+                let word_bytes = <Self::ABI as ABI>::word_bytes() as u32;
+                self.asm.push_r(reg);
+                self.increment_sp(word_bytes);
+                word_bytes
+            }
+            (RegClass::Int, OperandSize::S32) => {
+                let bytes = size.bytes();
+                self.reserve_stack(bytes)?;
+                let sp_offset = SPOffset::from_u32(self.sp_offset);
+                self.asm
+                    .mov_rm(reg, &self.address_from_sp(sp_offset)?, size, TRUSTED_FLAGS);
+                bytes
+            }
+            (RegClass::Float, _) => {
+                let bytes = size.bytes();
+                self.reserve_stack(bytes)?;
+                let sp_offset = SPOffset::from_u32(self.sp_offset);
+                self.asm
+                    .xmm_mov_rm(reg, &self.address_from_sp(sp_offset)?, size, TRUSTED_FLAGS);
+                bytes
+            }
+            _ => unreachable!(),
+        };
+
+        Ok(StackSlot {
+            offset: SPOffset::from_u32(self.sp_offset),
+            size: bytes,
+        })
+    }
+
+    fn reserve_stack(&mut self, bytes: u32) -> Result<()> {
+        if bytes == 0 {
+            return Ok(());
+        }
+
+        self.asm
+            .sub_ir(bytes as i32, writable!(rsp()), OperandSize::S64);
+        self.increment_sp(bytes);
+
+        Ok(())
+    }
+
+    fn free_stack(&mut self, bytes: u32) -> Result<()> {
+        if bytes == 0 {
+            return Ok(());
+        }
+        self.asm
+            .add_ir(bytes as i32, writable!(rsp()), OperandSize::S64);
+        self.decrement_sp(bytes);
+
+        Ok(())
+    }
+
+    fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()> {
+        self.sp_offset = offset.as_u32();
+
+        Ok(())
+    }
+
+    fn local_address(&mut self, local: &LocalSlot) -> Result<Address> {
+        let (reg, offset) = if local.addressed_from_sp() {
+            let offset = self
+                .sp_offset
+                .checked_sub(local.offset)
+                .ok_or_else(|| CodeGenError::invalid_local_offset())?;
+            (rsp(), offset)
+        } else {
+            (rbp(), local.offset)
+        };
+
+        Ok(Address::offset(reg, offset))
+    }
+
+    fn address_from_sp(&self, offset: SPOffset) -> Result<Self::Address> {
+        Ok(Address::offset(
+            regs::rsp(),
+            self.sp_offset - offset.as_u32(),
+        ))
+    }
+
+    fn address_at_sp(&self, offset: SPOffset) -> Result<Self::Address> {
+        Ok(Address::offset(regs::rsp(), offset.as_u32()))
+    }
+
+    fn address_at_vmctx(&self, offset: u32) -> Result<Self::Address> {
+        Ok(Address::offset(vmctx!(Self), offset))
+    }
+
+    fn store_ptr(&mut self, src: Reg, dst: Self::Address) -> Result<()> {
+        self.store(src.into(), dst, self.ptr_size)
+    }
+
+    fn store(&mut self, src: RegImm, dst: Address, size: OperandSize) -> Result<()> {
+        self.store_impl(src, dst, size, TRUSTED_FLAGS)
+    }
+
+    fn wasm_store(&mut self, src: Reg, dst: Self::Address, kind: StoreKind) -> Result<()> {
+        match kind {
+            StoreKind::Operand(size) => {
+                self.store_impl(src.into(), dst, size, UNTRUSTED_FLAGS)?;
+            }
+            StoreKind::Atomic(size) => {
+                if size == OperandSize::S128 {
+                    // TODO: we don't support 128-bit atomic store yet.
+                    bail!(CodeGenError::unexpected_operand_size());
+                }
+                // To stay consistent with cranelift, we emit a normal store followed by a mfence,
+                // although, we could probably just emit a xchg.
+                self.store_impl(src.into(), dst, size, UNTRUSTED_FLAGS)?;
+                self.asm.mfence();
+            }
+            StoreKind::VectorLane(LaneSelector { lane, size }) => {
+                self.ensure_has_avx()?;
+                self.asm
+                    .xmm_vpextr_rm(&dst, src, lane, size, UNTRUSTED_FLAGS);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn pop(&mut self, dst: WritableReg, size: OperandSize) -> Result<()> {
+        let current_sp = SPOffset::from_u32(self.sp_offset);
+        let _ = match (dst.to_reg().class(), size) {
+            (RegClass::Int, OperandSize::S32) => {
+                let addr = self.address_from_sp(current_sp)?;
+                self.asm.movzx_mr(
+                    &addr,
+                    dst,
+                    size.extend_to::<Zero>(OperandSize::S64),
+                    TRUSTED_FLAGS,
+                );
+                self.free_stack(size.bytes())?;
+            }
+            (RegClass::Int, OperandSize::S64) => {
+                self.asm.pop_r(dst);
+                self.decrement_sp(<Self::ABI as ABI>::word_bytes() as u32);
+            }
+            (RegClass::Float, _) | (RegClass::Vector, _) => {
+                let addr = self.address_from_sp(current_sp)?;
+                self.asm.xmm_mov_mr(&addr, dst, size, TRUSTED_FLAGS);
+                self.free_stack(size.bytes())?;
+            }
+            _ => bail!(CodeGenError::invalid_operand_combination()),
+        };
+        Ok(())
+    }
+
+    fn with_scratch<T: ScratchType, R>(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R {
+        let r = self
+            .scratch_scope
+            .reg_for_class(T::reg_class(), &mut |_| Ok(()))
+            .expect("Scratch register to be available");
+
+        let ret = f(self, Scratch::new(r));
+        self.scratch_scope.free(r);
+        ret
+    }
+
+    fn call(
+        &mut self,
+        stack_args_size: u32,
+        mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>,
+    ) -> Result<u32> {
+        let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into();
+        let addend: u32 = <Self::ABI as abi::ABI>::initial_frame_size().into();
+        let delta = calculate_frame_adjustment(self.sp_offset()?.as_u32(), addend, alignment);
+        let aligned_args_size = align_to(stack_args_size, alignment);
+        let total_stack = delta + aligned_args_size;
+        self.reserve_stack(total_stack)?;
+        let (callee, cc) = load_callee(self)?;
+        match callee {
+            CalleeKind::Indirect(reg) => self.asm.call_with_reg(cc, reg),
+            CalleeKind::Direct(idx) => self.asm.call_with_name(cc, idx),
+        };
+        Ok(total_stack)
+    }
+
+    fn load_ptr(&mut self, src: Self::Address, dst: WritableReg) -> Result<()> {
+        self.load(src, dst, self.ptr_size)
+    }
+
+    fn compute_addr(
+        &mut self,
+        src: Self::Address,
+        dst: WritableReg,
+        size: OperandSize,
+    ) -> Result<()> {
+        self.asm.lea(&src, dst, size);
+        Ok(())
+    }
+
+    fn load(&mut self, src: Address, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.load_impl(src, dst, size, TRUSTED_FLAGS)
+    }
+
+    fn wasm_load(&mut self, src: Self::Address, dst: WritableReg, kind: LoadKind) -> Result<()> {
+        let size = kind.derive_operand_size();
+
+        match kind {
+            LoadKind::ScalarExtend(ext) => match ext {
+                ExtendKind::Signed(ext) => {
+                    self.asm.movsx_mr(&src, dst, ext, UNTRUSTED_FLAGS);
+                }
+                ExtendKind::Unsigned(_) => self.load_impl(src, dst, size, UNTRUSTED_FLAGS)?,
+            },
+            LoadKind::Operand(_) | LoadKind::Atomic(_, _) => {
+                // The guarantees of the x86-64 memory model ensure that `SeqCst`
+                // loads are equivalent to normal loads.
+                if kind.is_atomic() && size == OperandSize::S128 {
+                    bail!(CodeGenError::unexpected_operand_size());
+                }
+
+                self.load_impl(src, dst, size, UNTRUSTED_FLAGS)?;
+            }
+            LoadKind::VectorExtend(ext) => {
+                self.ensure_has_avx()?;
+                self.asm
+                    .xmm_vpmov_mr(&src, dst, ext.into(), UNTRUSTED_FLAGS)
+            }
+            LoadKind::Splat(_) => {
+                self.ensure_has_avx()?;
+
+                if size == OperandSize::S64 {
+                    self.asm
+                        .xmm_mov_mr(&src, dst, OperandSize::S64, UNTRUSTED_FLAGS);
+                    self.asm.xmm_vpshuf_rr(
+                        dst.to_reg(),
+                        dst,
+                        Self::vpshuf_mask_for_64_bit_splats(),
+                        OperandSize::S32,
+                    );
+                } else {
+                    self.asm
+                        .xmm_vpbroadcast_mr(&src, dst, size, UNTRUSTED_FLAGS);
+                }
+            }
+            LoadKind::VectorLane(LaneSelector { lane, size }) => {
+                self.ensure_has_avx()?;
+                self.with_scratch::<IntScratch, _>(|masm, byte_tmp| {
+                    masm.load_impl(src, byte_tmp.writable(), size, UNTRUSTED_FLAGS)?;
+                    masm.asm
+                        .xmm_vpinsr_rrr(dst, dst.to_reg(), byte_tmp.inner(), lane, size);
+                    anyhow::Ok(())
+                })?;
+            }
+            LoadKind::VectorZero(size) => {
+                self.ensure_has_avx()?;
+                self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                    masm.load_impl(src, scratch.writable(), size, UNTRUSTED_FLAGS)?;
+                    masm.asm.avx_gpr_to_xmm(scratch.inner(), dst, size);
+                    anyhow::Ok(())
+                })?;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn sp_offset(&self) -> Result<SPOffset> {
+        Ok(SPOffset::from_u32(self.sp_offset))
+    }
+
+    fn zero(&mut self, reg: WritableReg) -> Result<()> {
+        self.asm.xor_rr(
+            reg.to_reg(),
+            reg,
+            OperandSize::from_bytes(<Self::ABI>::word_bytes()),
+        );
+        Ok(())
+    }
+
+    fn mov(&mut self, dst: WritableReg, src: RegImm, size: OperandSize) -> Result<()> {
+        match (src, dst.to_reg()) {
+            (RegImm::Reg(src), dst_reg) => match (src.class(), dst_reg.class()) {
+                (RegClass::Int, RegClass::Int) => Ok(self.asm.mov_rr(src, dst, size)),
+                (RegClass::Float, RegClass::Float) => Ok(self.asm.xmm_mov_rr(src, dst, size)),
+                _ => bail!(CodeGenError::invalid_operand_combination()),
+            },
+            (RegImm::Imm(imm), _) => self.load_constant(&imm, dst, size),
+        }
+    }
+
+    fn cmov(
+        &mut self,
+        dst: WritableReg,
+        src: Reg,
+        cc: IntCmpKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        match (src.class(), dst.to_reg().class()) {
+            (RegClass::Int, RegClass::Int) => Ok(self.asm.cmov(src, dst, cc, size)),
+            (RegClass::Float, RegClass::Float) => Ok(self.asm.xmm_cmov(src, dst, cc, size)),
+            _ => Err(anyhow!(CodeGenError::invalid_operand_combination())),
+        }
+    }
+
+    fn add(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        match (rhs, dst) {
+            (RegImm::Imm(imm), _) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.add_ir(v, dst, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.add_rr(scratch.inner(), dst, size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+
+            (RegImm::Reg(src), dst) => {
+                self.asm.add_rr(src, dst, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn add_uextend(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        from_size: OperandSize,
+        size: OperandSize,
+    ) -> Result<()> {
+        assert!(size == OperandSize::S64);
+        assert!(from_size == OperandSize::S32 || from_size == OperandSize::S64);
+
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        if from_size == OperandSize::S32 && size == OperandSize::S64 {
+            self.extend(
+                writable!(rhs),
+                rhs,
+                ExtendKind::Unsigned(Extend::I64Extend32),
+            )?;
+        }
+
+        self.asm.add_rr(rhs, dst, size);
+
+        Ok(())
+    }
+
+    fn checked_uadd(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Imm,
+        size: OperandSize,
+        trap: TrapCode,
+    ) -> Result<()> {
+        self.add(dst, lhs, RegImm::Imm(rhs), size)?;
+        self.asm.trapif(CC::B, trap);
+        Ok(())
+    }
+
+    fn sub(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        match (rhs, dst) {
+            (RegImm::Imm(imm), reg) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.sub_ir(v, reg, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.sub_rr(scratch.inner(), reg, size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+
+            (RegImm::Reg(src), dst) => {
+                self.asm.sub_rr(src, dst, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn mul(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        match (rhs, dst) {
+            (RegImm::Imm(imm), _) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.mul_ir(v, dst, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.mul_rr(scratch.inner(), dst, size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+
+            (RegImm::Reg(src), dst) => {
+                self.asm.mul_rr(src, dst, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn float_add(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm.xmm_add_rr(rhs, dst, size);
+        Ok(())
+    }
+
+    fn float_sub(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm.xmm_sub_rr(rhs, dst, size);
+        Ok(())
+    }
+
+    fn float_mul(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm.xmm_mul_rr(rhs, dst, size);
+        Ok(())
+    }
+
+    fn float_div(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm.xmm_div_rr(rhs, dst, size);
+        Ok(())
+    }
+
+    fn float_min(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm.xmm_min_seq(rhs, dst, size);
+        Ok(())
+    }
+
+    fn float_max(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm.xmm_max_seq(rhs, dst, size);
+        Ok(())
+    }
+
+    fn float_copysign(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        size: OperandSize,
+    ) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        let sign_mask = match size {
+            OperandSize::S32 => I::I32(0x80000000),
+            OperandSize::S64 => I::I64(0x8000000000000000),
+            OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => {
+                bail!(CodeGenError::unexpected_operand_size())
+            }
+        };
+
+        self.with_scratch::<IntScratch, _>(|masm, scratch_gpr| {
+            masm.with_scratch::<FloatScratch, _>(|masm, scratch_xmm| {
+                masm.load_constant(&sign_mask, scratch_gpr.writable(), size)?;
+                masm.asm
+                    .gpr_to_xmm(scratch_gpr.inner(), scratch_xmm.writable(), size);
+
+                // Clear everything except sign bit in src.
+                masm.asm
+                    .xmm_and_rr(scratch_xmm.inner(), writable!(rhs), size);
+
+                // Clear sign bit in dst using scratch to store result. Then copy the
+                // result back to dst.
+                masm.asm
+                    .xmm_andn_rr(dst.to_reg(), scratch_xmm.writable(), size);
+                masm.asm.xmm_mov_rr(scratch_xmm.inner(), dst, size);
+
+                // Copy sign bit from src to dst.
+                masm.asm.xmm_or_rr(rhs, dst, size);
+                Ok(())
+            })
+        })
+    }
+
+    fn float_neg(&mut self, dst: WritableReg, size: OperandSize) -> Result<()> {
+        debug_assert_eq!(dst.to_reg().class(), RegClass::Float);
+        let mask = match size {
+            OperandSize::S32 => I::I32(0x80000000),
+            OperandSize::S64 => I::I64(0x8000000000000000),
+            OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => {
+                bail!(CodeGenError::unexpected_operand_size())
+            }
+        };
+        self.with_scratch::<IntScratch, _>(|masm, scratch_gpr| {
+            masm.with_scratch::<FloatScratch, _>(|masm, scratch_xmm| {
+                masm.load_constant(&mask, scratch_gpr.writable(), size)?;
+                masm.asm
+                    .gpr_to_xmm(scratch_gpr.inner(), scratch_xmm.writable(), size);
+                masm.asm.xmm_xor_rr(scratch_xmm.inner(), dst, size);
+                Ok(())
+            })
+        })
+    }
+
+    fn float_abs(&mut self, dst: WritableReg, size: OperandSize) -> Result<()> {
+        debug_assert_eq!(dst.to_reg().class(), RegClass::Float);
+        let mask = match size {
+            OperandSize::S32 => I::I32(0x7fffffff),
+            OperandSize::S64 => I::I64(0x7fffffffffffffff),
+            OperandSize::S128 | OperandSize::S16 | OperandSize::S8 => {
+                bail!(CodeGenError::unexpected_operand_size())
+            }
+        };
+
+        self.with_scratch::<IntScratch, _>(|masm, scratch_gpr| {
+            masm.with_scratch::<FloatScratch, _>(|masm, scratch_xmm| {
+                masm.load_constant(&mask, scratch_gpr.writable(), size)?;
+
+                masm.asm
+                    .gpr_to_xmm(scratch_gpr.inner(), scratch_xmm.writable(), size);
+                masm.asm.xmm_and_rr(scratch_xmm.inner(), dst, size);
+                Ok(())
+            })
+        })
+    }
+
+    fn float_round<
+        F: FnMut(&mut FuncEnv<Self::Ptr>, &mut CodeGenContext<Emission>, &mut Self) -> Result<()>,
+    >(
+        &mut self,
+        mode: RoundingMode,
+        env: &mut FuncEnv<Self::Ptr>,
+        context: &mut CodeGenContext<Emission>,
+        size: OperandSize,
+        mut fallback: F,
+    ) -> Result<()> {
+        if self.flags.has_sse41() {
+            let src = context.pop_to_reg(self, None)?;
+            self.asm
+                .xmm_rounds_rr(src.into(), writable!(src.into()), mode, size);
+            context.stack.push(src.into());
+            Ok(())
+        } else {
+            fallback(env, context, self)
+        }
+    }
+
+    fn float_sqrt(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()> {
+        self.asm.sqrt(src, dst, size);
+        Ok(())
+    }
+
+    fn and(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        match (rhs, dst) {
+            (RegImm::Imm(imm), _) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.and_ir(v, dst, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.and_rr(scratch.inner(), dst, size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+
+            (RegImm::Reg(src), dst) => {
+                self.asm.and_rr(src, dst, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn or(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        match (rhs, dst) {
+            (RegImm::Imm(imm), _) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.or_ir(v, dst, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.or_rr(scratch.inner(), dst, size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+
+            (RegImm::Reg(src), dst) => {
+                self.asm.or_rr(src, dst, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn xor(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        match (rhs, dst) {
+            (RegImm::Imm(imm), _) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.xor_ir(v, dst, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.xor_rr(scratch.inner(), dst, size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+
+            (RegImm::Reg(src), _) => {
+                self.asm.xor_rr(src, dst, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn shift_ir(
+        &mut self,
+        dst: WritableReg,
+        imm: I,
+        lhs: Reg,
+        kind: ShiftKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        Self::ensure_two_argument_form(&dst.to_reg(), &lhs)?;
+        self.asm
+            .shift_ir(imm.unwrap_as_u64() as u8, dst, kind, size);
+        Ok(())
+    }
+
+    fn shift(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: ShiftKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        // Number of bits to shift must be in the CL register.
+        let src = context.pop_to_reg(self, Some(regs::rcx()))?;
+        let dst = context.pop_to_reg(self, None)?;
+
+        self.asm
+            .shift_rr(src.into(), writable!(dst.into()), kind, size);
+
+        context.free_reg(src);
+        context.stack.push(dst.into());
+
+        Ok(())
+    }
+
+    fn div(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: DivKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        // Allocate rdx:rax.
+        let rdx = context.reg(regs::rdx(), self)?;
+        let rax = context.reg(regs::rax(), self)?;
+
+        // Allocate the divisor, which can be any gpr.
+        let divisor = context.pop_to_reg(self, None)?;
+
+        // Mark rax as allocatable.
+        context.free_reg(rax);
+        // Move the top value to rax.
+        let rax = context.pop_to_reg(self, Some(rax))?;
+        self.asm.div(divisor.into(), (rax.into(), rdx), kind, size);
+
+        // Free the divisor and rdx.
+        context.free_reg(divisor);
+        context.free_reg(rdx);
+
+        // Push the quotient.
+        context.stack.push(rax.into());
+        Ok(())
+    }
+
+    fn rem(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: RemKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        // Allocate rdx:rax.
+        let rdx = context.reg(regs::rdx(), self)?;
+        let rax = context.reg(regs::rax(), self)?;
+
+        // Allocate the divisor, which can be any gpr.
+        let divisor = context.pop_to_reg(self, None)?;
+
+        // Mark rax as allocatable.
+        context.free_reg(rax);
+        // Move the top value to rax.
+        let rax = context.pop_to_reg(self, Some(rax))?;
+        self.asm.rem(divisor.reg, (rax.into(), rdx), kind, size);
+
+        // Free the divisor and rax.
+        context.free_reg(divisor);
+        context.free_reg(rax);
+
+        // Push the remainder.
+        context.stack.push(Val::reg(rdx, divisor.ty));
+
+        Ok(())
+    }
+
+    fn frame_restore(&mut self) -> Result<()> {
+        debug_assert_eq!(self.sp_offset, 0);
+        self.asm.pop_r(writable!(rbp()));
+        self.asm.ret();
+        Ok(())
+    }
+
+    fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>> {
+        if let Some(patch) = self.stack_max_use_add {
+            patch.finalize(i32::try_from(self.sp_max).unwrap(), self.asm.buffer_mut());
+        }
+
+        Ok(self.asm.finalize(base))
+    }
+
+    fn address_at_reg(&self, reg: Reg, offset: u32) -> Result<Self::Address> {
+        Ok(Address::offset(reg, offset))
+    }
+
+    fn cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize) -> Result<()> {
+        match src2 {
+            RegImm::Imm(imm) => {
+                if let Some(v) = imm.to_i32() {
+                    self.asm.cmp_ir(src1, v, size);
+                } else {
+                    self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                        masm.load_constant(&imm, scratch.writable(), size)?;
+                        masm.asm.cmp_rr(src1, scratch.inner(), size);
+                        anyhow::Ok(())
+                    })?;
+                }
+            }
+            RegImm::Reg(src2) => {
+                self.asm.cmp_rr(src1, src2, size);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn cmp_with_set(
+        &mut self,
+        dst: WritableReg,
+        src: RegImm,
+        kind: IntCmpKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        self.cmp(dst.to_reg(), src, size)?;
+        self.asm.setcc(kind, dst);
+        Ok(())
+    }
+
+    fn float_cmp_with_set(
+        &mut self,
+        dst: WritableReg,
+        src1: Reg,
+        src2: Reg,
+        kind: FloatCmpKind,
+        size: OperandSize,
+    ) -> Result<()> {
+        // Float comparisons needs to be ordered (that is, comparing with a NaN
+        // should return 0) except for not equal which needs to be unordered.
+        // We use ucomis{s, d} because comis{s, d} has an undefined result if
+        // either operand is NaN. Since ucomis{s, d} is unordered, we need to
+        // compensate to make the comparison ordered.  Ucomis{s, d} sets the
+        // ZF, PF, and CF flags if there is an unordered result.
+        let (src1, src2, set_kind) = match kind {
+            FloatCmpKind::Eq => (src1, src2, IntCmpKind::Eq),
+            FloatCmpKind::Ne => (src1, src2, IntCmpKind::Ne),
+            FloatCmpKind::Gt => (src1, src2, IntCmpKind::GtU),
+            FloatCmpKind::Ge => (src1, src2, IntCmpKind::GeU),
+            // Reversing the operands and using the complementary comparison
+            // avoids needing to perform an additional SETNP and AND
+            // instruction.
+            // SETNB and SETNBE check if the carry flag is unset (i.e., not
+            // less than and not unordered) so we get the intended result
+            // without having to look at the parity flag.
+            FloatCmpKind::Lt => (src2, src1, IntCmpKind::GtU),
+            FloatCmpKind::Le => (src2, src1, IntCmpKind::GeU),
+        };
+        self.asm.ucomis(src1, src2, size);
+        self.asm.setcc(set_kind, dst);
+        let _ = match kind {
+            FloatCmpKind::Eq | FloatCmpKind::Gt | FloatCmpKind::Ge => {
+                // Return false if either operand is NaN by ensuring PF is
+                // unset.
+                self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                    masm.asm.setnp(scratch.writable());
+                    masm.asm.and_rr(scratch.inner(), dst, size);
+                });
+            }
+            FloatCmpKind::Ne => {
+                // Return true if either operand is NaN by checking if PF is
+                // set.
+                self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                    masm.asm.setp(scratch.writable());
+                    masm.asm.or_rr(scratch.inner(), dst, size);
+                });
+            }
+            FloatCmpKind::Lt | FloatCmpKind::Le => (),
+        };
+        Ok(())
+    }
+
+    fn clz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()> {
+        if self.flags.has_lzcnt() {
+            self.asm.lzcnt(src, dst, size);
+        } else {
+            self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                // Use the following approach:
+                // dst = size.num_bits() - bsr(src) - is_not_zero
+                //     = size.num.bits() + -bsr(src) - is_not_zero.
+                masm.asm.bsr(src, dst, size);
+                masm.asm.setcc(IntCmpKind::Ne, scratch.writable());
+                masm.asm.neg(dst.to_reg(), dst, size);
+                masm.asm.add_ir(size.num_bits() as i32, dst, size);
+                masm.asm.sub_rr(scratch.inner(), dst, size);
+            });
+        }
+
+        Ok(())
+    }
+
+    fn ctz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()> {
+        if self.flags.has_bmi1() {
+            self.asm.tzcnt(src, dst, size);
+        } else {
+            self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                // Use the following approach:
+                // dst = bsf(src) + (is_zero * size.num_bits())
+                //     = bsf(src) + (is_zero << size.log2()).
+                // BSF outputs the correct value for every value except 0.
+                // When the value is 0, BSF outputs 0, correct output for ctz is
+                // the number of bits.
+                masm.asm.bsf(src, dst, size);
+                masm.asm.setcc(IntCmpKind::Eq, scratch.writable());
+                masm.asm
+                    .shift_ir(size.log2(), scratch.writable(), ShiftKind::Shl, size);
+                masm.asm.add_rr(scratch.inner(), dst, size);
+            });
+        }
+
+        Ok(())
+    }
+
+    fn get_label(&mut self) -> Result<MachLabel> {
+        let buffer = self.asm.buffer_mut();
+        Ok(buffer.get_label())
+    }
+
+    fn bind(&mut self, label: MachLabel) -> Result<()> {
+        let buffer = self.asm.buffer_mut();
+        buffer.bind_label(label, &mut Default::default());
+        Ok(())
+    }
+
+    fn branch(
+        &mut self,
+        kind: IntCmpKind,
+        lhs: Reg,
+        rhs: RegImm,
+        taken: MachLabel,
+        size: OperandSize,
+    ) -> Result<()> {
+        use IntCmpKind::*;
+
+        match &(lhs, rhs) {
+            (rlhs, RegImm::Reg(rrhs)) => {
+                // If the comparison kind is zero or not zero and both operands
+                // are the same register, emit a test instruction. Else we emit
+                // a normal comparison.
+                if (kind == Eq || kind == Ne) && (rlhs == rrhs) {
+                    self.asm.test_rr(*rlhs, *rrhs, size);
+                } else {
+                    self.cmp(lhs, rhs, size)?;
+                }
+            }
+            _ => self.cmp(lhs, rhs, size)?,
+        }
+        self.asm.jmp_if(kind, taken);
+        Ok(())
+    }
+
+    fn jmp(&mut self, target: MachLabel) -> Result<()> {
+        self.asm.jmp(target);
+        Ok(())
+    }
+
+    fn popcnt(&mut self, context: &mut CodeGenContext<Emission>, size: OperandSize) -> Result<()> {
+        let src = context.pop_to_reg(self, None)?;
+        if self.flags.has_popcnt() && self.flags.has_sse42() {
+            self.asm.popcnt(src.into(), writable!(src.into()), size);
+            context.stack.push(src.into());
+            Ok(())
+        } else {
+            // The fallback functionality here is based on `MacroAssembler::popcnt64` in:
+            // https://searchfox.org/mozilla-central/source/js/src/jit/x64/MacroAssembler-x64-inl.h#495
+
+            let tmp = writable!(context.any_gpr(self)?);
+            let dst = writable!(src.into());
+            let (masks, shift_amt) = match size {
+                OperandSize::S64 => (
+                    [
+                        0x5555555555555555, // m1
+                        0x3333333333333333, // m2
+                        0x0f0f0f0f0f0f0f0f, // m4
+                        0x0101010101010101, // h01
+                    ],
+                    56u8,
+                ),
+                // 32-bit popcount is the same, except the masks are half as
+                // wide and we shift by 24 at the end rather than 56
+                OperandSize::S32 => (
+                    [0x55555555i64, 0x33333333i64, 0x0f0f0f0fi64, 0x01010101i64],
+                    24u8,
+                ),
+                _ => bail!(CodeGenError::unexpected_operand_size()),
+            };
+            self.asm.mov_rr(src.into(), tmp, size);
+
+            // x -= (x >> 1) & m1;
+            self.asm.shift_ir(1u8, dst, ShiftKind::ShrU, size);
+            let lhs = dst.to_reg();
+            self.and(writable!(lhs), lhs, RegImm::i64(masks[0]), size)?;
+            self.asm.sub_rr(dst.to_reg(), tmp, size);
+
+            // x = (x & m2) + ((x >> 2) & m2);
+            self.asm.mov_rr(tmp.to_reg(), dst, size);
+            // Load `0x3333...` into the scratch reg once, allowing us to use
+            // `and_rr` and avoid inadvertently loading it twice as with `and`
+
+            self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                masm.load_constant(&I::i64(masks[1]), scratch.writable(), size)?;
+                masm.asm.and_rr(scratch.inner(), dst, size);
+                masm.asm.shift_ir(2u8, tmp, ShiftKind::ShrU, size);
+                masm.asm.and_rr(scratch.inner(), tmp, size);
+                anyhow::Ok(())
+            })?;
+            self.asm.add_rr(dst.to_reg(), tmp, size);
+
+            // x = (x + (x >> 4)) & m4;
+            self.asm.mov_rr(tmp.to_reg(), dst, size);
+            self.asm.shift_ir(4u8, dst, ShiftKind::ShrU, size);
+            self.asm.add_rr(tmp.to_reg(), dst, size);
+            let lhs = dst.to_reg();
+            self.and(writable!(lhs), lhs, RegImm::i64(masks[2]), size)?;
+
+            // (x * h01) >> shift_amt
+            let lhs = dst.to_reg();
+            self.mul(writable!(lhs), lhs, RegImm::i64(masks[3]), size)?;
+            self.asm.shift_ir(shift_amt, dst, ShiftKind::ShrU, size);
+
+            context.stack.push(src.into());
+            context.free_reg(tmp.to_reg());
+
+            Ok(())
+        }
+    }
+
+    fn wrap(&mut self, dst: WritableReg, src: Reg) -> Result<()> {
+        self.asm.mov_rr(src, dst, OperandSize::S32);
+        Ok(())
+    }
+
+    fn extend(&mut self, dst: WritableReg, src: Reg, kind: ExtendKind) -> Result<()> {
+        match kind {
+            ExtendKind::Signed(ext) => {
+                self.asm.movsx_rr(src, dst, ext);
+            }
+            ExtendKind::Unsigned(ext) => {
+                self.asm.movzx_rr(src, dst, ext);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn signed_truncate(
+        &mut self,
+        dst: WritableReg,
+        src: Reg,
+        src_size: OperandSize,
+        dst_size: OperandSize,
+        kind: TruncKind,
+    ) -> Result<()> {
+        self.with_scratch::<IntScratch, _>(|masm, gpr_scratch| {
+            masm.with_scratch::<FloatScratch, _>(|masm, xmm_scratch| {
+                masm.asm.cvt_float_to_sint_seq(
+                    src,
+                    dst,
+                    gpr_scratch.inner(),
+                    xmm_scratch.inner(),
+                    src_size,
+                    dst_size,
+                    kind.is_checked(),
+                );
+                Ok(())
+            })
+        })
+    }
+
+    fn unsigned_truncate(
+        &mut self,
+        ctx: &mut CodeGenContext<Emission>,
+        src_size: OperandSize,
+        dst_size: OperandSize,
+        kind: TruncKind,
+    ) -> Result<()> {
+        let dst_ty = match dst_size {
+            OperandSize::S32 => WasmValType::I32,
+            OperandSize::S64 => WasmValType::I64,
+            _ => bail!(CodeGenError::unexpected_operand_size()),
+        };
+
+        ctx.convert_op_with_tmp_reg(
+            self,
+            dst_ty,
+            RegClass::Float,
+            |masm, dst, src, tmp_fpr, dst_size| {
+                masm.with_scratch::<IntScratch, _>(|masm, gpr_scratch| {
+                    masm.with_scratch::<FloatScratch, _>(|masm, xmm_scratch| {
+                        masm.asm.cvt_float_to_uint_seq(
+                            src,
+                            writable!(dst),
+                            gpr_scratch.inner(),
+                            xmm_scratch.inner(),
+                            tmp_fpr,
+                            src_size,
+                            dst_size,
+                            kind.is_checked(),
+                        );
+                        Ok(())
+                    })
+                })
+            },
+        )
+    }
+
+    fn signed_convert(
+        &mut self,
+        dst: WritableReg,
+        src: Reg,
+        src_size: OperandSize,
+        dst_size: OperandSize,
+    ) -> Result<()> {
+        self.asm.cvt_sint_to_float(src, dst, src_size, dst_size);
+        Ok(())
+    }
+
+    fn unsigned_convert(
+        &mut self,
+        dst: WritableReg,
+        src: Reg,
+        tmp_gpr: Reg,
+        src_size: OperandSize,
+        dst_size: OperandSize,
+    ) -> Result<()> {
+        // Need to convert unsigned uint32 to uint64 for conversion instruction sequence.
+        if let OperandSize::S32 = src_size {
+            self.extend(
+                writable!(src),
+                src,
+                ExtendKind::Unsigned(Extend::I64Extend32),
+            )?;
+        }
+
+        self.with_scratch::<IntScratch, _>(|masm, scratch| {
+            masm.asm
+                .cvt_uint64_to_float_seq(src, dst, scratch.inner(), tmp_gpr, dst_size);
+            Ok(())
+        })
+    }
+
+    fn reinterpret_float_as_int(
+        &mut self,
+        dst: WritableReg,
+        src: Reg,
+        size: OperandSize,
+    ) -> Result<()> {
+        self.asm.xmm_to_gpr(src, dst, size);
+        Ok(())
+    }
+
+    fn reinterpret_int_as_float(
+        &mut self,
+        dst: WritableReg,
+        src: Reg,
+        size: OperandSize,
+    ) -> Result<()> {
+        self.asm.gpr_to_xmm(src, dst, size);
+        Ok(())
+    }
+
+    fn demote(&mut self, dst: WritableReg, src: Reg) -> Result<()> {
+        self.asm
+            .cvt_float_to_float(src, dst, OperandSize::S64, OperandSize::S32);
+        Ok(())
+    }
+
+    fn promote(&mut self, dst: WritableReg, src: Reg) -> Result<()> {
+        self.asm
+            .cvt_float_to_float(src, dst, OperandSize::S32, OperandSize::S64);
+        Ok(())
+    }
+
+    fn unreachable(&mut self) -> Result<()> {
+        self.asm.trap(TRAP_UNREACHABLE);
+        Ok(())
+    }
+
+    fn trap(&mut self, code: TrapCode) -> Result<()> {
+        self.asm.trap(code);
+        Ok(())
+    }
+
+    fn trapif(&mut self, cc: IntCmpKind, code: TrapCode) -> Result<()> {
+        self.asm.trapif(cc, code);
+        Ok(())
+    }
+
+    fn trapz(&mut self, src: Reg, code: TrapCode) -> Result<()> {
+        self.asm.test_rr(src, src, self.ptr_size);
+        self.asm.trapif(IntCmpKind::Eq, code);
+        Ok(())
+    }
+
+    fn jmp_table(&mut self, targets: &[MachLabel], index: Reg, tmp: Reg) -> Result<()> {
+        // At least one default target.
+        debug_assert!(targets.len() >= 1);
+        let default_index = targets.len() - 1;
+        // Emit bounds check, by conditionally moving the max cases
+        // into the given index reg if the contents of the index reg
+        // are greater.
+        let max = default_index;
+        let size = OperandSize::S32;
+        self.asm.mov_ir(max as u64, writable!(tmp), size);
+        self.asm.cmp_rr(tmp, index, size);
+        self.asm.cmov(tmp, writable!(index), IntCmpKind::LtU, size);
+
+        let default = targets[default_index];
+        let rest = &targets[0..default_index];
+
+        self.with_scratch::<IntScratch, _>(|masm, tmp1| {
+            masm.asm
+                .jmp_table(rest.into(), default, index, tmp1.inner(), tmp);
+            Ok(())
+        })
+    }
+
+    fn start_source_loc(&mut self, loc: RelSourceLoc) -> Result<(CodeOffset, RelSourceLoc)> {
+        Ok(self.asm.buffer_mut().start_srcloc(loc))
+    }
+
+    fn end_source_loc(&mut self) -> Result<()> {
+        self.asm.buffer_mut().end_srcloc();
+        Ok(())
+    }
+
+    fn current_code_offset(&self) -> Result<CodeOffset> {
+        Ok(self.asm.buffer().cur_offset())
+    }
+
+    fn add128(
+        &mut self,
+        dst_lo: WritableReg,
+        dst_hi: WritableReg,
+        lhs_lo: Reg,
+        lhs_hi: Reg,
+        rhs_lo: Reg,
+        rhs_hi: Reg,
+    ) -> Result<()> {
+        Self::ensure_two_argument_form(&dst_lo.to_reg(), &lhs_lo)?;
+        Self::ensure_two_argument_form(&dst_hi.to_reg(), &lhs_hi)?;
+        self.asm.add_rr(rhs_lo, dst_lo, OperandSize::S64);
+        self.asm.adc_rr(rhs_hi, dst_hi, OperandSize::S64);
+        Ok(())
+    }
+
+    fn sub128(
+        &mut self,
+        dst_lo: WritableReg,
+        dst_hi: WritableReg,
+        lhs_lo: Reg,
+        lhs_hi: Reg,
+        rhs_lo: Reg,
+        rhs_hi: Reg,
+    ) -> Result<()> {
+        Self::ensure_two_argument_form(&dst_lo.to_reg(), &lhs_lo)?;
+        Self::ensure_two_argument_form(&dst_hi.to_reg(), &lhs_hi)?;
+        self.asm.sub_rr(rhs_lo, dst_lo, OperandSize::S64);
+        self.asm.sbb_rr(rhs_hi, dst_hi, OperandSize::S64);
+        Ok(())
+    }
+
+    fn mul_wide(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: MulWideKind,
+    ) -> Result<()> {
+        // Reserve rax/rdx since they're required by the `mul_wide` instruction
+        // being used here.
+        let rax = context.reg(regs::rax(), self)?;
+        let rdx = context.reg(regs::rdx(), self)?;
+
+        // The rhs of this binop can be in any register
+        let rhs = context.pop_to_reg(self, None)?;
+        // Mark rax as allocatable. and then force the lhs operand to be placed
+        // in `rax`.
+        context.free_reg(rax);
+        let lhs = context.pop_to_reg(self, Some(rax))?;
+
+        self.asm.mul_wide(
+            writable!(rax),
+            writable!(rdx),
+            lhs.reg,
+            rhs.reg,
+            kind,
+            OperandSize::S64,
+        );
+
+        // No longer using the rhs register after the multiplication has been
+        // executed.
+        context.free_reg(rhs);
+
+        // The low bits of the result are in rax, where `lhs` was allocated to
+        context.stack.push(lhs.into());
+        // The high bits of the result are in rdx, which we previously reserved.
+        context.stack.push(Val::Reg(TypedReg::i64(rdx)));
+
+        Ok(())
+    }
+
+    fn splat(&mut self, context: &mut CodeGenContext<Emission>, size: SplatKind) -> Result<()> {
+        // Get the source and destination operands set up first.
+        let (src, dst) = match size {
+            // Floats can use the same register for `src` and `dst`.
+            SplatKind::F32x4 | SplatKind::F64x2 => {
+                let reg = context.pop_to_reg(self, None)?.reg;
+                (RegImm::reg(reg), writable!(reg))
+            }
+            // For ints, we need to load the operand into a vector register if
+            // it's not a constant.
+            SplatKind::I8x16 | SplatKind::I16x8 | SplatKind::I32x4 | SplatKind::I64x2 => {
+                let dst = writable!(context.any_fpr(self)?);
+                let src = if size == SplatKind::I64x2 {
+                    context.pop_i64_const().map(RegImm::i64)
+                } else {
+                    context.pop_i32_const().map(RegImm::i32)
+                }
+                .map_or_else(
+                    || -> Result<RegImm> {
+                        let reg = context.pop_to_reg(self, None)?.reg;
+                        self.reinterpret_int_as_float(
+                            dst,
+                            reg,
+                            match size {
+                                SplatKind::I8x16 | SplatKind::I16x8 | SplatKind::I32x4 => {
+                                    OperandSize::S32
+                                }
+                                SplatKind::I64x2 => OperandSize::S64,
+                                SplatKind::F32x4 | SplatKind::F64x2 => unreachable!(),
+                            },
+                        )?;
+                        context.free_reg(reg);
+                        Ok(RegImm::Reg(dst.to_reg()))
+                    },
+                    Ok,
+                )?;
+                (src, dst)
+            }
+        };
+
+        // Perform the splat on the operands.
+        if size == SplatKind::I64x2 || size == SplatKind::F64x2 {
+            self.ensure_has_avx()?;
+            let mask = Self::vpshuf_mask_for_64_bit_splats();
+            match src {
+                RegImm::Reg(src) => self.asm.xmm_vpshuf_rr(src, dst, mask, OperandSize::S32),
+                RegImm::Imm(imm) => {
+                    let src = self.asm.add_constant(&imm.to_bytes());
+                    self.asm
+                        .xmm_vpshuf_mr(&src, dst, mask, OperandSize::S32, MemFlags::trusted());
+                }
+            }
+        } else {
+            self.ensure_has_avx2()?;
+
+            match src {
+                RegImm::Reg(src) => self.asm.xmm_vpbroadcast_rr(src, dst, size.lane_size()),
+                RegImm::Imm(imm) => {
+                    let src = self.asm.add_constant(&imm.to_bytes());
+                    self.asm
+                        .xmm_vpbroadcast_mr(&src, dst, size.lane_size(), MemFlags::trusted());
+                }
+            }
+        }
+
+        context
+            .stack
+            .push(Val::reg(dst.to_reg(), WasmValType::V128));
+        Ok(())
+    }
+
+    fn shuffle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, lanes: [u8; 16]) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        // Use `vpshufb` with `lanes` to set the lanes in `lhs` and `rhs`
+        // separately to either the selected index or 0.
+        // Then use `vpor` to combine `lhs` and `rhs` into `dst`.
+        // Setting the most significant bit in the mask's lane to 1 will
+        // result in corresponding lane in the destination register being
+        // set to 0. 0x80 sets the most significant bit to 1.
+        let mut mask_lhs: [u8; 16] = [0x80; 16];
+        let mut mask_rhs: [u8; 16] = [0x80; 16];
+        for i in 0..lanes.len() {
+            if lanes[i] < 16 {
+                mask_lhs[i] = lanes[i];
+            } else {
+                mask_rhs[i] = lanes[i] - 16;
+            }
+        }
+        let mask_lhs = self.asm.add_constant(&mask_lhs);
+        let mask_rhs = self.asm.add_constant(&mask_rhs);
+
+        self.asm.xmm_vpshufb_rrm(dst, lhs, &mask_lhs);
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            masm.asm.xmm_vpshufb_rrm(scratch.writable(), rhs, &mask_rhs);
+            masm.asm.xmm_vpor_rrr(dst, dst.to_reg(), scratch.inner());
+            Ok(())
+        })
+    }
+
+    fn swizzle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        // Clamp rhs to [0, 15 (i.e., 0xF)] and substitute 0 for anything
+        // outside that range.
+        // Each lane is a signed byte so the maximum value is 0x7F. Adding
+        // 0x70 to any value higher than 0xF will saturate resulting in a value
+        // of 0xFF (i.e., 0).
+        let clamp = self.asm.add_constant(&[0x70; 16]);
+        self.asm
+            .xmm_vpaddus_rrm(writable!(rhs), rhs, &clamp, OperandSize::S8);
+
+        // Don't need to subtract 0x70 since `vpshufb` uses the least
+        // significant 4 bits which are the same after adding 0x70.
+        self.asm.xmm_vpshufb_rrr(dst, lhs, rhs);
+        Ok(())
+    }
+
+    fn atomic_rmw(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        addr: Self::Address,
+        size: OperandSize,
+        op: RmwOp,
+        flags: MemFlags,
+        extend: Option<Extend<Zero>>,
+    ) -> Result<()> {
+        let res = match op {
+            RmwOp::Add => {
+                let operand = context.pop_to_reg(self, None)?;
+                self.asm
+                    .lock_xadd(addr, writable!(operand.reg), size, flags);
+                operand.reg
+            }
+            RmwOp::Sub => {
+                let operand = context.pop_to_reg(self, None)?;
+                self.asm.neg(operand.reg, writable!(operand.reg), size);
+                self.asm
+                    .lock_xadd(addr, writable!(operand.reg), size, flags);
+                operand.reg
+            }
+            RmwOp::Xchg => {
+                let operand = context.pop_to_reg(self, None)?;
+                self.asm.xchg(addr, writable!(operand.reg), size, flags);
+                operand.reg
+            }
+            RmwOp::And | RmwOp::Or | RmwOp::Xor => {
+                let op = match op {
+                    RmwOp::And => AtomicRmwSeqOp::And,
+                    RmwOp::Or => AtomicRmwSeqOp::Or,
+                    RmwOp::Xor => AtomicRmwSeqOp::Xor,
+                    _ => unreachable!(
+                        "invalid op for atomic_rmw_seq, should be one of `or`, `and` or `xor`"
+                    ),
+                };
+                let dst = context.reg(regs::rax(), self)?;
+                let operand = context.pop_to_reg(self, None)?;
+
+                self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                    masm.asm.atomic_rmw_seq(
+                        addr,
+                        operand.reg,
+                        writable!(dst),
+                        scratch.writable(),
+                        size,
+                        flags,
+                        op,
+                    );
+                });
+
+                context.free_reg(operand.reg);
+                dst
+            }
+        };
+
+        let dst_ty = match extend {
+            Some(ext) => {
+                // We don't need to zero-extend from 32 to 64bits.
+                if !(ext.from_bits() == 32 && ext.to_bits() == 64) {
+                    self.asm.movzx_rr(res, writable!(res), ext);
+                }
+
+                WasmValType::int_from_bits(ext.to_bits())
+            }
+            None => WasmValType::int_from_bits(size.num_bits()),
+        };
+
+        context.stack.push(TypedReg::new(dst_ty, res).into());
+
+        Ok(())
+    }
+
+    fn extract_lane(
+        &mut self,
+        src: Reg,
+        dst: WritableReg,
+        lane: u8,
+        kind: ExtractLaneKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            ExtractLaneKind::I8x16S
+            | ExtractLaneKind::I8x16U
+            | ExtractLaneKind::I16x8S
+            | ExtractLaneKind::I16x8U
+            | ExtractLaneKind::I32x4
+            | ExtractLaneKind::I64x2 => self.asm.xmm_vpextr_rr(dst, src, lane, kind.lane_size()),
+            ExtractLaneKind::F32x4 | ExtractLaneKind::F64x2 if lane == 0 => {
+                // If the `src` and `dst` registers are the same, then the
+                // appropriate value is already in the correct position in
+                // the register.
+                assert!(src == dst.to_reg());
+            }
+            ExtractLaneKind::F32x4 => self.asm.xmm_vpshuf_rr(src, dst, lane, kind.lane_size()),
+            ExtractLaneKind::F64x2 => {
+                // `0b11_10` selects the high and low 32-bits of the second
+                // 64-bit, so `0b11_10_11_10` splats the 64-bit value across
+                // both lanes. Since we put an `f64` on the stack, we use
+                // the splatted value.
+                // Double-check `lane == 0` was handled in another branch.
+                assert!(lane == 1);
+                self.asm
+                    .xmm_vpshuf_rr(src, dst, 0b11_10_11_10, OperandSize::S32)
+            }
+        }
+
+        // Sign-extend to 32-bits for sign extended kinds.
+        match kind {
+            ExtractLaneKind::I8x16S | ExtractLaneKind::I16x8S => {
+                self.asm.movsx_rr(dst.to_reg(), dst, kind.into())
+            }
+            _ => (),
+        }
+
+        Ok(())
+    }
+
+    fn replace_lane(
+        &mut self,
+        src: RegImm,
+        dst: WritableReg,
+        lane: u8,
+        kind: ReplaceLaneKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            ReplaceLaneKind::I8x16
+            | ReplaceLaneKind::I16x8
+            | ReplaceLaneKind::I32x4
+            | ReplaceLaneKind::I64x2 => match src {
+                RegImm::Reg(reg) => {
+                    self.asm
+                        .xmm_vpinsr_rrr(dst, dst.to_reg(), reg, lane, kind.lane_size());
+                }
+                RegImm::Imm(imm) => {
+                    let address = self.asm.add_constant(&imm.to_bytes());
+                    self.asm
+                        .xmm_vpinsr_rrm(dst, dst.to_reg(), &address, lane, kind.lane_size());
+                }
+            },
+            ReplaceLaneKind::F32x4 => {
+                // Immediate for `vinsertps` uses first 3 bits to determine
+                // which elements of the destination to set to 0. The next 2
+                // bits specify which element of the destination will be
+                // overwritten.
+                let imm = lane << 4;
+                match src {
+                    RegImm::Reg(reg) => self.asm.xmm_vinsertps_rrr(dst, dst.to_reg(), reg, imm),
+                    RegImm::Imm(val) => {
+                        let address = self.asm.add_constant(&val.to_bytes());
+                        self.asm.xmm_vinsertps_rrm(dst, dst.to_reg(), &address, imm);
+                    }
+                }
+            }
+            ReplaceLaneKind::F64x2 => match src {
+                RegImm::Reg(reg) => match lane {
+                    0 => self.asm.xmm_vmovsd_rrr(dst, dst.to_reg(), reg),
+                    1 => self.asm.xmm_vmovlhps_rrr(dst, dst.to_reg(), reg),
+                    _ => unreachable!(),
+                },
+                RegImm::Imm(imm) => {
+                    let address = self.asm.add_constant(&imm.to_bytes());
+                    match lane {
+                        0 => {
+                            // Memory load variant of `vmovsd` zeroes the upper
+                            // 64 bits of the register so need to load the
+                            // immediate to a register to use the register
+                            // variant of `vmovsd` to perform the merge.
+
+                            self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                                masm.asm.xmm_vmovsd_rm(scratch.writable(), &address);
+                                masm.asm.xmm_vmovsd_rrr(dst, dst.to_reg(), scratch.inner());
+                            });
+                        }
+                        1 => self.asm.xmm_vmovlhps_rrm(dst, dst.to_reg(), &address),
+                        _ => unreachable!(),
+                    }
+                }
+            },
+        }
+        Ok(())
+    }
+
+    fn atomic_cas(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        addr: Self::Address,
+        size: OperandSize,
+        flags: MemFlags,
+        extend: Option<Extend<Zero>>,
+    ) -> Result<()> {
+        // `cmpxchg` expects `expected` to be in the `*a*` register.
+        // reserve rax for the expected argument.
+        let rax = context.reg(regs::rax(), self)?;
+
+        let replacement = context.pop_to_reg(self, None)?;
+
+        // mark `rax` as allocatable again.
+        context.free_reg(rax);
+        let expected = context.pop_to_reg(self, Some(regs::rax()))?;
+
+        self.asm
+            .cmpxchg(addr, replacement.reg, writable!(expected.reg), size, flags);
+
+        if let Some(extend) = extend {
+            // We don't need to zero-extend from 32 to 64bits.
+            if !(extend.from_bits() == 32 && extend.to_bits() == 64) {
+                self.asm
+                    .movzx_rr(expected.reg, writable!(expected.reg), extend);
+            }
+        }
+
+        context.stack.push(expected.into());
+        context.free_reg(replacement);
+
+        Ok(())
+    }
+
+    fn v128_eq(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        kind: VectorEqualityKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            VectorEqualityKind::I8x16
+            | VectorEqualityKind::I16x8
+            | VectorEqualityKind::I32x4
+            | VectorEqualityKind::I64x2 => {
+                self.asm.xmm_vpcmpeq_rrr(dst, lhs, rhs, kind.lane_size())
+            }
+            VectorEqualityKind::F32x4 | VectorEqualityKind::F64x2 => {
+                self.asm
+                    .xmm_vcmpp_rrr(dst, lhs, rhs, kind.lane_size(), VcmpKind::Eq)
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_ne(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        kind: VectorEqualityKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            VectorEqualityKind::I8x16
+            | VectorEqualityKind::I16x8
+            | VectorEqualityKind::I32x4
+            | VectorEqualityKind::I64x2 => {
+                // Check for equality and invert the results.
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(lhs), lhs, rhs, kind.lane_size());
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(rhs), rhs, rhs, kind.lane_size());
+                self.asm.xmm_vpxor_rrr(lhs, rhs, dst);
+            }
+            VectorEqualityKind::F32x4 | VectorEqualityKind::F64x2 => {
+                self.asm
+                    .xmm_vcmpp_rrr(dst, lhs, rhs, kind.lane_size(), VcmpKind::Ne)
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_lt(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        kind: VectorCompareKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            VectorCompareKind::I8x16S
+            | VectorCompareKind::I16x8S
+            | VectorCompareKind::I32x4S
+            | VectorCompareKind::I64x2S => {
+                // Perform a greater than check with reversed parameters.
+                self.asm.xmm_vpcmpgt_rrr(dst, rhs, lhs, kind.lane_size())
+            }
+            VectorCompareKind::I8x16U | VectorCompareKind::I16x8U | VectorCompareKind::I32x4U => {
+                // Set `lhs` to min values, check for equality, then invert the
+                // result.
+                // If `lhs` is smaller, then equality check will fail and result
+                // will be inverted to true. Otherwise the equality check will
+                // pass and be inverted to false.
+                self.asm
+                    .xmm_vpminu_rrr(writable!(lhs), lhs, rhs, kind.lane_size());
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(lhs), lhs, rhs, kind.lane_size());
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(rhs), rhs, rhs, kind.lane_size());
+                self.asm.xmm_vpxor_rrr(lhs, rhs, dst);
+            }
+            VectorCompareKind::F32x4 | VectorCompareKind::F64x2 => {
+                self.asm
+                    .xmm_vcmpp_rrr(dst, lhs, rhs, kind.lane_size(), VcmpKind::Lt)
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_le(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        kind: VectorCompareKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            VectorCompareKind::I8x16S | VectorCompareKind::I16x8S | VectorCompareKind::I32x4S => {
+                // Set the `rhs` vector to the signed minimum values and then
+                // compare them with `lhs` for equality.
+                self.asm
+                    .xmm_vpmins_rrr(writable!(rhs), lhs, rhs, kind.lane_size());
+                self.asm.xmm_vpcmpeq_rrr(dst, lhs, rhs, kind.lane_size());
+            }
+            VectorCompareKind::I64x2S => {
+                // Do a greater than check and invert the results.
+                self.asm
+                    .xmm_vpcmpgt_rrr(writable!(lhs), lhs, rhs, kind.lane_size());
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(rhs), rhs, rhs, kind.lane_size());
+                self.asm.xmm_vpxor_rrr(lhs, rhs, dst);
+            }
+            VectorCompareKind::I8x16U | VectorCompareKind::I16x8U | VectorCompareKind::I32x4U => {
+                // Set the `rhs` vector to the signed minimum values and then
+                // compare them with `lhs` for equality.
+                self.asm
+                    .xmm_vpminu_rrr(writable!(rhs), lhs, rhs, kind.lane_size());
+                self.asm.xmm_vpcmpeq_rrr(dst, lhs, rhs, kind.lane_size());
+            }
+            VectorCompareKind::F32x4 | VectorCompareKind::F64x2 => {
+                self.asm
+                    .xmm_vcmpp_rrr(dst, lhs, rhs, kind.lane_size(), VcmpKind::Le)
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_gt(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        kind: VectorCompareKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            VectorCompareKind::I8x16S
+            | VectorCompareKind::I16x8S
+            | VectorCompareKind::I32x4S
+            | VectorCompareKind::I64x2S => {
+                self.asm.xmm_vpcmpgt_rrr(dst, lhs, rhs, kind.lane_size())
+            }
+            VectorCompareKind::I8x16U | VectorCompareKind::I16x8U | VectorCompareKind::I32x4U => {
+                // Set `lhs` to max values, check for equality, then invert the
+                // result.
+                // If `lhs` is larger, then equality check will fail and result
+                // will be inverted to true. Otherwise the equality check will
+                // pass and be inverted to false.
+                self.asm
+                    .xmm_vpmaxu_rrr(writable!(lhs), lhs, rhs, kind.lane_size());
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(lhs), lhs, rhs, kind.lane_size());
+                self.asm
+                    .xmm_vpcmpeq_rrr(writable!(rhs), rhs, rhs, kind.lane_size());
+                self.asm.xmm_vpxor_rrr(lhs, rhs, dst);
+            }
+            VectorCompareKind::F32x4 | VectorCompareKind::F64x2 => {
+                // Do a less than comparison with the operands swapped.
+                self.asm
+                    .xmm_vcmpp_rrr(dst, rhs, lhs, kind.lane_size(), VcmpKind::Lt)
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_ge(
+        &mut self,
+        dst: WritableReg,
+        lhs: Reg,
+        rhs: Reg,
+        kind: VectorCompareKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            VectorCompareKind::I8x16S | VectorCompareKind::I16x8S | VectorCompareKind::I32x4S => {
+                // Set each lane to maximum value and then compare for equality.
+                self.asm
+                    .xmm_vpmaxs_rrr(writable!(rhs), lhs, rhs, kind.lane_size());
+                self.asm.xmm_vpcmpeq_rrr(dst, lhs, rhs, kind.lane_size());
+            }
+            VectorCompareKind::I64x2S => {
+                // Perform a greater than comparison with operands swapped,
+                // then invert the results.
+                self.asm
+                    .xmm_vpcmpgt_rrr(writable!(rhs), rhs, lhs, kind.lane_size());
+                self.asm.xmm_vpcmpeq_rrr(dst, lhs, lhs, kind.lane_size());
+                self.asm.xmm_vpxor_rrr(dst.to_reg(), rhs, dst);
+            }
+            VectorCompareKind::I8x16U | VectorCompareKind::I16x8U | VectorCompareKind::I32x4U => {
+                // Set lanes to maximum values and compare them for equality.
+                self.asm
+                    .xmm_vpmaxu_rrr(writable!(rhs), lhs, rhs, kind.lane_size());
+                self.asm.xmm_vpcmpeq_rrr(dst, lhs, rhs, kind.lane_size());
+            }
+            VectorCompareKind::F32x4 | VectorCompareKind::F64x2 => {
+                // Perform a less than or equal comparison on swapped operands.
+                self.asm
+                    .xmm_vcmpp_rrr(dst, rhs, lhs, kind.lane_size(), VcmpKind::Le)
+            }
+        }
+
+        Ok(())
+    }
+
+    fn fence(&mut self) -> Result<()> {
+        self.asm.mfence();
+        Ok(())
+    }
+
+    fn v128_not(&mut self, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        self.with_scratch::<FloatScratch, _>(|masm, tmp| {
+            // First, we initialize `tmp` with all ones by comparing it with
+            // itself.
+            masm.asm
+                .xmm_vpcmpeq_rrr(tmp.writable(), tmp.inner(), tmp.inner(), OperandSize::S32);
+            // Then we `xor` tmp and `dst` together, yielding `!dst`.
+            masm.asm.xmm_vpxor_rrr(tmp.inner(), dst.to_reg(), dst);
+            Ok(())
+        })
+    }
+
+    fn v128_and(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vpand_rrr(src1, src2, dst);
+        Ok(())
+    }
+
+    fn v128_and_not(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vpandn_rrr(src1, src2, dst);
+        Ok(())
+    }
+
+    fn v128_or(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vpor_rrr(dst, src1, src2);
+        Ok(())
+    }
+
+    fn v128_xor(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vpxor_rrr(src1, src2, dst);
+        Ok(())
+    }
+
+    fn v128_bitselect(&mut self, src1: Reg, src2: Reg, mask: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        self.with_scratch::<FloatScratch, _>(|masm, tmp| {
+            masm.v128_and(src1, mask, tmp.writable())?;
+            masm.v128_and_not(mask, src2, dst)?;
+            masm.v128_or(dst.to_reg(), tmp.inner(), dst)?;
+            Ok(())
+        })
+    }
+
+    fn v128_any_true(&mut self, src: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vptest(src, src);
+        self.asm.setcc(IntCmpKind::Ne, dst);
+        Ok(())
+    }
+
+    fn v128_convert(&mut self, src: Reg, dst: WritableReg, kind: V128ConvertKind) -> Result<()> {
+        self.ensure_has_avx()?;
+        match kind {
+            V128ConvertKind::I32x4S => self.asm.xmm_vcvt_rr(src, dst, VcvtKind::I32ToF32),
+            V128ConvertKind::I32x4LowS => self.asm.xmm_vcvt_rr(src, dst, VcvtKind::I32ToF64),
+            V128ConvertKind::I32x4U => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Split each 32-bit integer into 16-bit parts.
+                    // `scratch` will contain the low bits and `dst` will contain
+                    // the high bits.
+                    masm.asm
+                        .xmm_vpsll_rri(src, scratch.writable(), 0x10, kind.src_lane_size());
+                    masm.asm.xmm_vpsrl_rri(
+                        scratch.inner(),
+                        scratch.writable(),
+                        0x10,
+                        kind.src_lane_size(),
+                    );
+                    masm.asm
+                        .xmm_vpsub_rrr(src, scratch.inner(), dst, kind.src_lane_size());
+
+                    // Convert the low bits in `scratch` to floating point numbers.
+                    masm.asm
+                        .xmm_vcvt_rr(scratch.inner(), scratch.writable(), VcvtKind::I32ToF32);
+
+                    // Prevent overflow by right shifting high bits.
+                    masm.asm
+                        .xmm_vpsrl_rri(dst.to_reg(), dst, 1, kind.src_lane_size());
+                    // Convert high bits in `dst` to floating point numbers.
+                    masm.asm.xmm_vcvt_rr(dst.to_reg(), dst, VcvtKind::I32ToF32);
+                    // Double high bits in `dst` to reverse right shift.
+                    masm.asm
+                        .xmm_vaddp_rrr(dst.to_reg(), dst.to_reg(), dst, kind.src_lane_size());
+                    // Add high bits in `dst` to low bits in `scratch`.
+                    masm.asm.xmm_vaddp_rrr(
+                        dst.to_reg(),
+                        scratch.inner(),
+                        dst,
+                        kind.src_lane_size(),
+                    );
+                });
+            }
+            V128ConvertKind::I32x4LowU => {
+                // See
+                // https://github.com/bytecodealliance/wasmtime/blob/bb886ffc3c81a476d8ba06311ff2dede15a6f7e1/cranelift/codegen/src/isa/x64/lower.isle#L3668
+                // for details on the Cranelift AVX implementation.
+                // Use `vunpcklp` to create doubles from the integers.
+                // Interleaving 0x1.0p52 (i.e., 0x43300000) with the integers
+                // creates a byte array for a double that sets the mantissa
+                // bits to the original integer value.
+                let conversion_constant = self
+                    .asm
+                    .add_constant(&[0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x30, 0x43]);
+                self.asm
+                    .xmm_vunpcklp_rrm(src, &conversion_constant, dst, kind.src_lane_size());
+                // Subtract the 0x1.0p52 added above.
+                let conversion_constant = self.asm.add_constant(&[
+                    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00,
+                    0x00, 0x30, 0x43,
+                ]);
+                self.asm.xmm_vsub_rrm(
+                    dst.to_reg(),
+                    &conversion_constant,
+                    dst,
+                    kind.dst_lane_size(),
+                );
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_narrow(
+        &mut self,
+        src1: Reg,
+        src2: Reg,
+        dst: WritableReg,
+        kind: V128NarrowKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+        match kind {
+            V128NarrowKind::I16x8S | V128NarrowKind::I32x4S => {
+                self.asm
+                    .xmm_vpackss_rrr(src1, src2, dst, kind.dst_lane_size())
+            }
+            V128NarrowKind::I16x8U | V128NarrowKind::I32x4U => {
+                self.asm
+                    .xmm_vpackus_rrr(src1, src2, dst, kind.dst_lane_size())
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_demote(&mut self, src: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vcvt_rr(src, dst, VcvtKind::F64ToF32);
+        Ok(())
+    }
+
+    fn v128_promote(&mut self, src: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vcvt_rr(src, dst, VcvtKind::F32ToF64);
+        Ok(())
+    }
+
+    fn v128_extend(&mut self, src: Reg, dst: WritableReg, kind: V128ExtendKind) -> Result<()> {
+        self.ensure_has_avx()?;
+        match kind {
+            V128ExtendKind::LowI8x16S
+            | V128ExtendKind::LowI8x16U
+            | V128ExtendKind::LowI16x8S
+            | V128ExtendKind::LowI16x8U
+            | V128ExtendKind::LowI32x4S
+            | V128ExtendKind::LowI32x4U => self.asm.xmm_vpmov_rr(src, dst, kind.into()),
+            V128ExtendKind::HighI8x16S | V128ExtendKind::HighI16x8S => {
+                self.asm.xmm_vpalignr_rrr(src, src, dst, 0x8);
+                self.asm.xmm_vpmov_rr(dst.to_reg(), dst, kind.into());
+            }
+            V128ExtendKind::HighI8x16U | V128ExtendKind::HighI16x8U => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    masm.asm
+                        .xmm_vpxor_rrr(scratch.inner(), scratch.inner(), scratch.writable());
+                    masm.asm
+                        .xmm_vpunpckh_rrr(src, scratch.inner(), dst, kind.src_lane_size());
+                });
+            }
+            V128ExtendKind::HighI32x4S => {
+                // Move the 3rd element (i.e., 0b10) to the 1st (rightmost)
+                // position and the 4th element (i.e., 0b11) to the 2nd (second
+                // from the right) position and then perform the extend.
+                self.asm
+                    .xmm_vpshuf_rr(src, dst, 0b11_10_11_10, kind.src_lane_size());
+                self.asm.xmm_vpmov_rr(dst.to_reg(), dst, kind.into());
+            }
+            V128ExtendKind::HighI32x4U => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Set `scratch` to a vector 0s.
+                    masm.asm.xmm_vxorp_rrr(
+                        scratch.inner(),
+                        scratch.inner(),
+                        scratch.writable(),
+                        kind.src_lane_size(),
+                    );
+                    // Interleave the 0 bits into the two 32-bit integers to zero extend them.
+                    masm.asm
+                        .xmm_vunpckhp_rrr(src, scratch.inner(), dst, kind.src_lane_size());
+                });
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_add(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, kind: V128AddKind) -> Result<()> {
+        self.ensure_has_avx()?;
+        match kind {
+            V128AddKind::F32x4 => self.asm.xmm_vaddp_rrr(lhs, rhs, dst, OperandSize::S32),
+            V128AddKind::F64x2 => self.asm.xmm_vaddp_rrr(lhs, rhs, dst, OperandSize::S64),
+            V128AddKind::I8x16 => self.asm.xmm_vpadd_rrr(lhs, rhs, dst, OperandSize::S8),
+            V128AddKind::I8x16SatS => self.asm.xmm_vpadds_rrr(dst, lhs, rhs, OperandSize::S8),
+            V128AddKind::I8x16SatU => self.asm.xmm_vpaddus_rrr(dst, lhs, rhs, OperandSize::S8),
+            V128AddKind::I16x8 => self.asm.xmm_vpadd_rrr(lhs, rhs, dst, OperandSize::S16),
+            V128AddKind::I16x8SatS => self.asm.xmm_vpadds_rrr(dst, lhs, rhs, OperandSize::S16),
+            V128AddKind::I16x8SatU => self.asm.xmm_vpaddus_rrr(dst, lhs, rhs, OperandSize::S16),
+            V128AddKind::I32x4 => self.asm.xmm_vpadd_rrr(lhs, rhs, dst, OperandSize::S32),
+            V128AddKind::I64x2 => self.asm.xmm_vpadd_rrr(lhs, rhs, dst, OperandSize::S64),
+        };
+        Ok(())
+    }
+
+    fn v128_sub(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, kind: V128SubKind) -> Result<()> {
+        self.ensure_has_avx()?;
+        match kind {
+            V128SubKind::F32x4 => self.asm.xmm_vsubp_rrr(lhs, rhs, dst, OperandSize::S32),
+            V128SubKind::F64x2 => self.asm.xmm_vsubp_rrr(lhs, rhs, dst, OperandSize::S64),
+            V128SubKind::I8x16 => self.asm.xmm_vpsub_rrr(lhs, rhs, dst, OperandSize::S8),
+            V128SubKind::I8x16SatS => self.asm.xmm_vpsubs_rrr(dst, lhs, rhs, OperandSize::S8),
+            V128SubKind::I8x16SatU => self.asm.xmm_vpsubus_rrr(dst, lhs, rhs, OperandSize::S8),
+            V128SubKind::I16x8 => self.asm.xmm_vpsub_rrr(lhs, rhs, dst, OperandSize::S16),
+            V128SubKind::I16x8SatS => self.asm.xmm_vpsubs_rrr(dst, lhs, rhs, OperandSize::S16),
+            V128SubKind::I16x8SatU => self.asm.xmm_vpsubus_rrr(dst, lhs, rhs, OperandSize::S16),
+            V128SubKind::I32x4 => self.asm.xmm_vpsub_rrr(lhs, rhs, dst, OperandSize::S32),
+            V128SubKind::I64x2 => self.asm.xmm_vpsub_rrr(lhs, rhs, dst, OperandSize::S64),
+        };
+        Ok(())
+    }
+
+    fn v128_mul(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: V128MulKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        let rhs = context.pop_to_reg(self, None)?;
+        let lhs = context.pop_to_reg(self, None)?;
+
+        let mul_i64x2_avx512 = |this: &mut Self| {
+            this.asm.vpmullq(lhs.reg, rhs.reg, writable!(lhs.reg));
+        };
+
+        let mul_i64x2_fallback = |this: &mut Self,
+                                  context: &mut CodeGenContext<Emission>|
+         -> Result<()> {
+            // Standard AVX doesn't have an instruction for i64x2 multiplication, instead, we have to fallback
+            // to an instruction sequence using 32bits multiplication (taken from cranelift
+            // implementation, in `isa/x64/lower.isle`):
+            //
+            // > Otherwise, for i64x2 multiplication we describe a lane A as being composed of
+            // > a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand
+            // > multiplication can then be written as:
+            //
+            // >    Ah Al
+            // > *  Bh Bl
+            // >    -----
+            // >    Al * Bl
+            // > + (Ah * Bl) << 32
+            // > + (Al * Bh) << 32
+            //
+            // > So for each lane we will compute:
+            //
+            // >   A * B  = (Al * Bl) + ((Ah * Bl) + (Al * Bh)) << 32
+            //
+            // > Note, the algorithm will use `pmuludq` which operates directly on the lower
+            // > 32-bit (`Al` or `Bl`) of a lane and writes the result to the full 64-bits of
+            // > the lane of the destination. For this reason we don't need shifts to isolate
+            // > the lower 32-bits, however, we will need to use shifts to isolate the high
+            // > 32-bits when doing calculations, i.e., `Ah == A >> 32`.
+
+            let tmp2 = context.any_fpr(this)?;
+            this.with_scratch::<FloatScratch, _>(|this, tmp1| {
+                // tmp1 = lhs_hi = (lhs >> 32)
+                this.asm
+                    .xmm_vpsrl_rri(lhs.reg, tmp1.writable(), 32, OperandSize::S64);
+
+                // tmp2 = lhs_hi * rhs_low = tmp1 * rhs
+                this.asm
+                    .xmm_vpmuldq_rrr(tmp1.inner(), rhs.reg, writable!(tmp2));
+
+                // tmp1 = rhs_hi = rhs >> 32
+                this.asm
+                    .xmm_vpsrl_rri(rhs.reg, tmp1.writable(), 32, OperandSize::S64);
+
+                // tmp1 = lhs_low * rhs_high = tmp1 * lhs
+                this.asm
+                    .xmm_vpmuludq_rrr(tmp1.inner(), lhs.reg, tmp1.writable());
+
+                // tmp1 = ((lhs_hi * rhs_low) + (lhs_lo * rhs_hi)) = tmp1 + tmp2
+                this.asm
+                    .xmm_vpadd_rrr(tmp1.inner(), tmp2, tmp1.writable(), OperandSize::S64);
+
+                //tmp1 = tmp1 << 32
+                this.asm
+                    .xmm_vpsll_rri(tmp1.inner(), tmp1.writable(), 32, OperandSize::S64);
+
+                // tmp2 = lhs_lo + rhs_lo
+                this.asm.xmm_vpmuludq_rrr(lhs.reg, rhs.reg, writable!(tmp2));
+
+                // finally, with `lhs` as destination:
+                // lhs = (lhs_low * rhs_low) + ((lhs_hi * rhs_low) + (lhs_lo * rhs_hi)) = tmp1 + tmp2
+                this.asm
+                    .xmm_vpadd_rrr(tmp1.inner(), tmp2, writable!(lhs.reg), OperandSize::S64);
+            });
+
+            context.free_reg(tmp2);
+
+            Ok(())
+        };
+
+        match kind {
+            V128MulKind::F32x4 => {
+                self.asm
+                    .xmm_vmulp_rrr(lhs.reg, rhs.reg, writable!(lhs.reg), OperandSize::S32)
+            }
+            V128MulKind::F64x2 => {
+                self.asm
+                    .xmm_vmulp_rrr(lhs.reg, rhs.reg, writable!(lhs.reg), OperandSize::S64)
+            }
+            V128MulKind::I16x8 => {
+                self.asm
+                    .xmm_vpmull_rrr(lhs.reg, rhs.reg, writable!(lhs.reg), OperandSize::S16)
+            }
+            V128MulKind::I32x4 => {
+                self.asm
+                    .xmm_vpmull_rrr(lhs.reg, rhs.reg, writable!(lhs.reg), OperandSize::S32)
+            }
+            // This is the fast path when AVX512 is available.
+            V128MulKind::I64x2
+                if self.ensure_has_avx512vl().is_ok() && self.ensure_has_avx512dq().is_ok() =>
+            {
+                mul_i64x2_avx512(self)
+            }
+            // Otherwise, we emit AVX fallback sequence.
+            V128MulKind::I64x2 => mul_i64x2_fallback(self, context)?,
+        }
+
+        context.stack.push(lhs.into());
+        context.free_reg(rhs);
+
+        Ok(())
+    }
+
+    fn v128_abs(&mut self, src: Reg, dst: WritableReg, kind: V128AbsKind) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            V128AbsKind::I8x16 | V128AbsKind::I16x8 | V128AbsKind::I32x4 => {
+                self.asm.xmm_vpabs_rr(src, dst, kind.lane_size())
+            }
+            V128AbsKind::I64x2 => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Perform an arithmetic right shift of 31 bits. If the number
+                    // is positive, this will result in all zeroes in the upper
+                    // 32-bits. If the number is negative, this will result in all
+                    // ones in the upper 32-bits.
+                    masm.asm
+                        .xmm_vpsra_rri(src, scratch.writable(), 0x1f, OperandSize::S32);
+                    // Copy the ones and zeroes in the high bits of each 64-bit
+                    // lane to the low bits of each 64-bit lane.
+                    masm.asm.xmm_vpshuf_rr(
+                        scratch.inner(),
+                        scratch.writable(),
+                        0b11_11_01_01,
+                        OperandSize::S32,
+                    );
+                    // Flip the bits in lanes that were negative in `src` and leave
+                    // the positive lanes as they are. Positive lanes will have a
+                    // zero mask in `scratch` so xor doesn't affect them.
+                    masm.asm.xmm_vpxor_rrr(src, scratch.inner(), dst);
+                    // Subtract the mask from the results of xor which will
+                    // complete the two's complement for lanes which were negative.
+                    masm.asm
+                        .xmm_vpsub_rrr(dst.to_reg(), scratch.inner(), dst, kind.lane_size());
+                });
+            }
+            V128AbsKind::F32x4 | V128AbsKind::F64x2 => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Create a mask of all ones.
+                    masm.asm.xmm_vpcmpeq_rrr(
+                        scratch.writable(),
+                        scratch.inner(),
+                        scratch.inner(),
+                        kind.lane_size(),
+                    );
+                    // Right shift the mask so each lane is a single zero followed
+                    // by all ones.
+                    masm.asm.xmm_vpsrl_rri(
+                        scratch.inner(),
+                        scratch.writable(),
+                        0x1,
+                        kind.lane_size(),
+                    );
+                    // Use the mask to zero the sign bit in each lane which will
+                    // make the float value positive.
+                    masm.asm
+                        .xmm_vandp_rrr(src, scratch.inner(), dst, kind.lane_size());
+                });
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_neg(&mut self, op: WritableReg, kind: V128NegKind) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            V128NegKind::I8x16 | V128NegKind::I16x8 | V128NegKind::I32x4 | V128NegKind::I64x2 => {
+                self.with_scratch::<FloatScratch, _>(|masm, tmp| {
+                    masm.v128_xor(tmp.inner(), tmp.inner(), tmp.writable())?;
+                    masm.v128_sub(tmp.inner(), op.to_reg(), op, kind.into())?;
+                    anyhow::Ok(())
+                })?;
+            }
+            V128NegKind::F32x4 | V128NegKind::F64x2 => {
+                self.with_scratch::<FloatScratch, _>(|masm, tmp| {
+                    // Create a mask of all 1s.
+                    masm.asm.xmm_vpcmpeq_rrr(
+                        tmp.writable(),
+                        tmp.inner(),
+                        tmp.inner(),
+                        kind.lane_size(),
+                    );
+                    // Left shift the lanes in the mask so only the sign bit in the
+                    // mask is set to 1.
+                    masm.asm.xmm_vpsll_rri(
+                        tmp.inner(),
+                        tmp.writable(),
+                        (kind.lane_size().num_bits() - 1) as u32,
+                        kind.lane_size(),
+                    );
+                    // Use the mask to flip the sign bit.
+                    masm.asm
+                        .xmm_vxorp_rrr(op.to_reg(), tmp.inner(), op, kind.lane_size());
+                });
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_shift(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        lane_width: OperandSize,
+        kind: ShiftKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+        let shift_amount = context.pop_to_reg(self, None)?.reg;
+        let operand = context.pop_to_reg(self, None)?.reg;
+        let amount_mask = lane_width.num_bits() - 1;
+
+        self.and(
+            writable!(shift_amount),
+            shift_amount,
+            RegImm::i32(amount_mask as i32),
+            OperandSize::S32,
+        )?;
+
+        self.with_scratch::<IntScratch, _>(|masm, tmp| {
+            masm.with_scratch::<FloatScratch, _>(|masm, tmp_xmm| {
+                let move_to_tmp_xmm = |this: &mut Self| {
+                    this.asm
+                        .avx_gpr_to_xmm(shift_amount, tmp_xmm.writable(), OperandSize::S32);
+                };
+
+                // A helper for deciding between `vpsllw` and `vpsrlw` in
+                // `shift_i8x16`.
+                enum Direction {
+                    Left,
+                    Right,
+                }
+
+                let shift_i8x16 = |this: &mut Self, masks: &'static [u8], direction: Direction| {
+                    // The case for i8x16 is a little bit trickier because x64 doesn't provide a 8bit
+                    // shift instruction. Instead, we shift as 16bits, and then mask the bits in the
+                    // 8bits lane, for example (with 2 8bits lanes):
+                    // - Before shifting:
+                    // 01001101 11101110
+                    // - shifting by 2 left:
+                    // 00110111 10111000
+                    //       ^^_ these bits come from the previous byte, and need to be masked.
+                    // - The mask:
+                    // 11111100 11111111
+                    // - After masking:
+                    // 00110100 10111000
+                    //
+                    // The mask is loaded from a well known memory, depending on the shift amount.
+
+                    this.asm
+                        .avx_gpr_to_xmm(shift_amount, tmp_xmm.writable(), OperandSize::S32);
+
+                    // Perform the 16-bit shift.
+                    match direction {
+                        Direction::Left => this.asm.xmm_vpsll_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S16,
+                        ),
+                        Direction::Right => this.asm.xmm_vpsrl_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S16,
+                        ),
+                    }
+
+                    // Get a handle to the masks array constant.
+                    let masks_addr = this.asm.add_constant(masks);
+
+                    // Load the masks array effective address into the tmp register.
+                    this.asm.lea(&masks_addr, tmp.writable(), OperandSize::S64);
+
+                    // Compute the offset of the mask that we need to use. This is shift_amount * 16 ==
+                    // shift_amount << 4.
+                    this.asm
+                        .shift_ir(4, writable!(shift_amount), ShiftKind::Shl, OperandSize::S32);
+
+                    // Load the mask to tmp_xmm.
+                    this.asm.xmm_vmovdqu_mr(
+                        &Address::ImmRegRegShift {
+                            simm32: 0,
+                            base: tmp.inner(),
+                            index: shift_amount,
+                            shift: 0,
+                        },
+                        tmp_xmm.writable(),
+                        MemFlags::trusted(),
+                    );
+
+                    // Mask unwanted bits from operand.
+                    this.asm
+                        .xmm_vpand_rrr(tmp_xmm.inner(), operand, writable!(operand));
+                };
+
+                let i64x2_shr_s = |this: &mut Self,
+                                   context: &mut CodeGenContext<Emission>|
+                 -> Result<()> {
+                    const SIGN_MASK: u128 = 0x8000000000000000_8000000000000000;
+
+                    // AVX doesn't have an instruction for i64x2 signed right shift. Instead we use the
+                    // following formula (from hacker's delight 2-7), where x is the value and n the shift
+                    // amount, for each lane:
+                    // t = (1 << 63) >> n; ((x >> n) ^ t) - t
+
+                    // We need an extra scratch register:
+                    let tmp_xmm2 = context.any_fpr(this)?;
+
+                    this.asm
+                        .avx_gpr_to_xmm(shift_amount, tmp_xmm.writable(), OperandSize::S32);
+
+                    let cst = this.asm.add_constant(&SIGN_MASK.to_le_bytes());
+
+                    this.asm
+                        .xmm_vmovdqu_mr(&cst, writable!(tmp_xmm2), MemFlags::trusted());
+                    this.asm.xmm_vpsrl_rrr(
+                        tmp_xmm2,
+                        tmp_xmm.inner(),
+                        writable!(tmp_xmm2),
+                        OperandSize::S64,
+                    );
+                    this.asm.xmm_vpsrl_rrr(
+                        operand,
+                        tmp_xmm.inner(),
+                        writable!(operand),
+                        OperandSize::S64,
+                    );
+                    this.asm
+                        .xmm_vpxor_rrr(operand, tmp_xmm2, writable!(operand));
+                    this.asm
+                        .xmm_vpsub_rrr(operand, tmp_xmm2, writable!(operand), OperandSize::S64);
+
+                    context.free_reg(tmp_xmm2);
+
+                    Ok(())
+                };
+
+                let i8x16_shr_s = |this: &mut Self,
+                                   context: &mut CodeGenContext<Emission>|
+                 -> Result<()> {
+                    // Since the x86 instruction set does not have an 8x16 shift instruction and the
+                    // approach used for `ishl` and `ushr` cannot be easily used (the masks do not
+                    // preserve the sign), we use a different approach here: separate the low and
+                    // high lanes, shift them separately, and merge them into the final result.
+                    //
+                    // Visually, this looks like the following, where `src.i8x16 = [s0, s1, ...,
+                    // s15]:
+                    //
+                    //   lo.i16x8 = [(s0, s0), (s1, s1), ..., (s7, s7)]
+                    //   shifted_lo.i16x8 = shift each lane of `low`
+                    //   hi.i16x8 = [(s8, s8), (s9, s9), ..., (s15, s15)]
+                    //   shifted_hi.i16x8 = shift each lane of `high`
+                    //   result = [s0'', s1'', ..., s15'']
+
+                    // In order for `packsswb` later to only use the high byte of each
+                    // 16x8 lane, we shift right an extra 8 bits, relying on `psraw` to
+                    // fill in the upper bits appropriately.
+                    this.asm
+                        .add_ir(8, writable!(shift_amount), OperandSize::S32);
+                    this.asm
+                        .avx_gpr_to_xmm(shift_amount, tmp_xmm.writable(), OperandSize::S32);
+
+                    let tmp_lo = context.any_fpr(this)?;
+                    let tmp_hi = context.any_fpr(this)?;
+
+                    // Extract lower and upper bytes.
+                    this.asm
+                        .xmm_vpunpckl_rrr(operand, operand, writable!(tmp_lo), OperandSize::S8);
+                    this.asm
+                        .xmm_vpunpckh_rrr(operand, operand, writable!(tmp_hi), OperandSize::S8);
+
+                    // Perform 16bit right shift of upper and lower bytes.
+                    this.asm.xmm_vpsra_rrr(
+                        tmp_lo,
+                        tmp_xmm.inner(),
+                        writable!(tmp_lo),
+                        OperandSize::S16,
+                    );
+                    this.asm.xmm_vpsra_rrr(
+                        tmp_hi,
+                        tmp_xmm.inner(),
+                        writable!(tmp_hi),
+                        OperandSize::S16,
+                    );
+
+                    // Merge lower and upper bytes back.
+                    this.asm
+                        .xmm_vpackss_rrr(tmp_lo, tmp_hi, writable!(operand), OperandSize::S8);
+
+                    context.free_reg(tmp_lo);
+                    context.free_reg(tmp_hi);
+
+                    Ok(())
+                };
+
+                match (lane_width, kind) {
+                    // shl
+                    (OperandSize::S8, ShiftKind::Shl) => {
+                        shift_i8x16(masm, &I8X16_ISHL_MASKS, Direction::Left)
+                    }
+                    (OperandSize::S16, ShiftKind::Shl) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsll_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S16,
+                        );
+                    }
+                    (OperandSize::S32, ShiftKind::Shl) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsll_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S32,
+                        );
+                    }
+                    (OperandSize::S64, ShiftKind::Shl) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsll_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S64,
+                        );
+                    }
+                    // shr_u
+                    (OperandSize::S8, ShiftKind::ShrU) => {
+                        shift_i8x16(masm, &I8X16_USHR_MASKS, Direction::Right)
+                    }
+                    (OperandSize::S16, ShiftKind::ShrU) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsrl_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S16,
+                        );
+                    }
+                    (OperandSize::S32, ShiftKind::ShrU) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsrl_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S32,
+                        );
+                    }
+                    (OperandSize::S64, ShiftKind::ShrU) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsrl_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S64,
+                        );
+                    }
+                    // shr_s
+                    (OperandSize::S8, ShiftKind::ShrS) => i8x16_shr_s(masm, context)?,
+                    (OperandSize::S16, ShiftKind::ShrS) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsra_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S16,
+                        );
+                    }
+                    (OperandSize::S32, ShiftKind::ShrS) => {
+                        move_to_tmp_xmm(masm);
+                        masm.asm.xmm_vpsra_rrr(
+                            operand,
+                            tmp_xmm.inner(),
+                            writable!(operand),
+                            OperandSize::S32,
+                        );
+                    }
+                    (OperandSize::S64, ShiftKind::ShrS) => i64x2_shr_s(masm, context)?,
+
+                    _ => bail!(CodeGenError::invalid_operand_combination()),
+                }
+
+                Ok(())
+            })
+        })?;
+
+        context.free_reg(shift_amount);
+        context
+            .stack
+            .push(TypedReg::new(WasmValType::V128, operand).into());
+        Ok(())
+    }
+
+    fn v128_q15mulr_sat_s(
+        &mut self,
+        lhs: Reg,
+        rhs: Reg,
+        dst: WritableReg,
+        size: OperandSize,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        self.asm.xmm_vpmulhrs_rrr(lhs, rhs, dst, size);
+
+        // Need to handle edge case of multiplying -1 by -1 (0x8000 in Q15
+        // format) because of how `vpmulhrs` handles rounding. `vpmulhrs`
+        // produces 0x8000 in that case when the correct result is 0x7FFF (that
+        // is, +1) so need to check if the result is 0x8000 and flip the bits
+        // of the result if it is.
+        let address = self.asm.add_constant(&[
+            0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80,
+            0x00, 0x80,
+        ]);
+        self.asm
+            .xmm_vpcmpeq_rrm(writable!(rhs), dst.to_reg(), &address, size);
+        self.asm.xmm_vpxor_rrr(dst.to_reg(), rhs, dst);
+        Ok(())
+    }
+
+    fn v128_all_true(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            // Create a mask of all 0s.
+            masm.asm
+                .xmm_vpxor_rrr(scratch.inner(), scratch.inner(), scratch.writable());
+            // Sets lane in `dst` to not zero if `src` lane was zero, and lane in
+            // `dst` to zero if `src` lane was not zero.
+            masm.asm
+                .xmm_vpcmpeq_rrr(writable!(src), src, scratch.inner(), size);
+            // Sets ZF if all values are zero (i.e., if all original values were not zero).
+            masm.asm.xmm_vptest(src, src);
+            // Set byte if ZF=1.
+        });
+        self.asm.setcc(IntCmpKind::Eq, dst);
+        Ok(())
+    }
+
+    fn v128_bitmask(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match size {
+            OperandSize::S8 => self.asm.xmm_vpmovmsk_rr(src, dst, size, OperandSize::S32),
+            OperandSize::S16 => {
+                // Signed conversion of 16-bit integers to 8-bit integers.
+                self.asm
+                    .xmm_vpackss_rrr(src, src, writable!(src), OperandSize::S8);
+                // Creates a mask from each byte in `src`.
+                self.asm
+                    .xmm_vpmovmsk_rr(src, dst, OperandSize::S8, OperandSize::S32);
+                // Removes 8 bits added as a result of the `vpackss` step.
+                self.asm
+                    .shift_ir(0x8, dst, ShiftKind::ShrU, OperandSize::S32);
+            }
+            OperandSize::S32 | OperandSize::S64 => {
+                self.asm.xmm_vmovskp_rr(src, dst, size, OperandSize::S32)
+            }
+            _ => unimplemented!(),
+        }
+
+        Ok(())
+    }
+
+    fn v128_trunc(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: V128TruncKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        let reg = writable!(context.pop_to_reg(self, None)?.reg);
+        match kind {
+            V128TruncKind::F32x4 | V128TruncKind::F64x2 => self.asm.xmm_vroundp_rri(
+                reg.to_reg(),
+                reg,
+                VroundMode::TowardZero,
+                kind.dst_lane_size(),
+            ),
+            V128TruncKind::I32x4FromF32x4S => {
+                self.v128_trunc_sat_f32x4_s(reg, kind.src_lane_size(), kind.dst_lane_size())?;
+            }
+            V128TruncKind::I32x4FromF32x4U => {
+                let temp_reg = writable!(context.any_fpr(self)?);
+                self.v128_trunc_sat_f32x4_u(
+                    reg,
+                    temp_reg,
+                    kind.src_lane_size(),
+                    kind.dst_lane_size(),
+                )?;
+                context.free_reg(temp_reg.to_reg());
+            }
+            V128TruncKind::I32x4FromF64x2SZero => {
+                self.v128_trunc_sat_f64x2_s_zero(reg, kind.src_lane_size())?;
+            }
+            V128TruncKind::I32x4FromF64x2UZero => {
+                self.v128_trunc_sat_f64x2_u_zero(reg, kind.src_lane_size(), kind.dst_lane_size())?;
+            }
+        }
+
+        context.stack.push(TypedReg::v128(reg.to_reg()).into());
+        Ok(())
+    }
+
+    fn v128_min(
+        &mut self,
+        src1: Reg,
+        src2: Reg,
+        dst: WritableReg,
+        kind: V128MinKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            V128MinKind::I8x16S
+            | V128MinKind::I8x16U
+            | V128MinKind::I16x8S
+            | V128MinKind::I16x8U
+            | V128MinKind::I32x4S
+            | V128MinKind::I32x4U => {
+                match kind {
+                    V128MinKind::I8x16S => {
+                        self.asm.xmm_vpmins_rrr(dst, src1, src2, OperandSize::S8)
+                    }
+                    V128MinKind::I8x16U => {
+                        self.asm.xmm_vpminu_rrr(dst, src1, src2, OperandSize::S8)
+                    }
+                    V128MinKind::I16x8S => {
+                        self.asm.xmm_vpmins_rrr(dst, src1, src2, OperandSize::S16)
+                    }
+                    V128MinKind::I16x8U => {
+                        self.asm.xmm_vpminu_rrr(dst, src1, src2, OperandSize::S16)
+                    }
+                    V128MinKind::I32x4S => {
+                        self.asm.xmm_vpmins_rrr(dst, src1, src2, OperandSize::S32)
+                    }
+                    V128MinKind::I32x4U => {
+                        self.asm.xmm_vpminu_rrr(dst, src1, src2, OperandSize::S32)
+                    }
+                    _ => unreachable!(),
+                };
+            }
+            V128MinKind::F32x4 | V128MinKind::F64x2 => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Handling +0 and -0 as well as NaN values are not commutative
+                    // when using `vminp` so we have to compensate.
+                    // Perform two comparison operations with the operands swapped
+                    // and OR the result to propagate 0 (positive and negative) and
+                    // NaN.
+                    masm.asm
+                        .xmm_vminp_rrr(src1, src2, scratch.writable(), kind.lane_size());
+                    masm.asm.xmm_vminp_rrr(src2, src1, dst, kind.lane_size());
+                    // Use a single OR instruction to set the sign bit if either
+                    // result has the sign bit set to correctly propagate -0.
+                    masm.asm
+                        .xmm_vorp_rrr(dst.to_reg(), scratch.inner(), dst, kind.lane_size());
+                });
+                // Set lanes with NaN to all 1s.
+                self.asm.xmm_vcmpp_rrr(
+                    writable!(src2),
+                    src2,
+                    dst.to_reg(),
+                    kind.lane_size(),
+                    VcmpKind::Unord,
+                );
+                // Doesn't change non-NaN values. For NaN values, sets all bits.
+                self.asm
+                    .xmm_vorp_rrr(src2, dst.to_reg(), dst, kind.lane_size());
+                self.canonicalize_nans(writable!(src2), dst, kind.lane_size());
+            }
+        }
+
+        Ok(())
+    }
+
+    fn v128_max(
+        &mut self,
+        src1: Reg,
+        src2: Reg,
+        dst: WritableReg,
+        kind: V128MaxKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            V128MaxKind::I8x16S
+            | V128MaxKind::I8x16U
+            | V128MaxKind::I16x8S
+            | V128MaxKind::I16x8U
+            | V128MaxKind::I32x4S
+            | V128MaxKind::I32x4U => {
+                match kind {
+                    V128MaxKind::I8x16S => {
+                        self.asm.xmm_vpmaxs_rrr(dst, src1, src2, OperandSize::S8)
+                    }
+                    V128MaxKind::I8x16U => {
+                        self.asm.xmm_vpmaxu_rrr(dst, src1, src2, OperandSize::S8)
+                    }
+                    V128MaxKind::I16x8S => {
+                        self.asm.xmm_vpmaxs_rrr(dst, src1, src2, OperandSize::S16)
+                    }
+                    V128MaxKind::I16x8U => {
+                        self.asm.xmm_vpmaxu_rrr(dst, src1, src2, OperandSize::S16)
+                    }
+                    V128MaxKind::I32x4S => {
+                        self.asm.xmm_vpmaxs_rrr(dst, src1, src2, OperandSize::S32)
+                    }
+                    V128MaxKind::I32x4U => {
+                        self.asm.xmm_vpmaxu_rrr(dst, src1, src2, OperandSize::S32)
+                    }
+                    _ => unreachable!(),
+                };
+            }
+            V128MaxKind::F32x4 | V128MaxKind::F64x2 => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Handling +0 and -0 as well as NaN values are not commutative
+                    // when using `vmaxp` so we have to compensate.
+                    // Perform two comparison operations with the operands swapped
+                    // so we can propagate 0 (positive and negative) and NaNs
+                    // correctly.
+
+                    masm.asm
+                        .xmm_vmaxp_rrr(src1, src2, scratch.writable(), kind.lane_size());
+                    masm.asm.xmm_vmaxp_rrr(src2, src1, dst, kind.lane_size());
+                    // This combination of XOR, OR, and SUB will set the sign bit
+                    // on a 0 result to the correct value for a max operation.
+                    masm.asm
+                        .xmm_vxorp_rrr(dst.to_reg(), scratch.inner(), dst, kind.lane_size());
+                    masm.asm.xmm_vorp_rrr(
+                        dst.to_reg(),
+                        scratch.inner(),
+                        writable!(src2),
+                        kind.lane_size(),
+                    );
+                });
+                self.asm
+                    .xmm_vsub_rrr(src2, dst.to_reg(), dst, kind.lane_size());
+                // Set lanes of NaN values to 1.
+                self.asm.xmm_vcmpp_rrr(
+                    writable!(src2),
+                    src2,
+                    src2,
+                    kind.lane_size(),
+                    VcmpKind::Unord,
+                );
+                self.canonicalize_nans(writable!(src2), dst, kind.lane_size());
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_extmul(
+        &mut self,
+        context: &mut CodeGenContext<Emission>,
+        kind: V128ExtMulKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        // The implementation for extmul is not optimized; for simplicity's sake, we simply perform
+        // an extension followed by a multiplication using already implemented primitives.
+
+        let src1 = context.pop_to_reg(self, None)?;
+        let src2 = context.pop_to_reg(self, None)?;
+
+        let ext_kind = kind.into();
+        self.v128_extend(src1.reg, writable!(src1.reg), ext_kind)?;
+        self.v128_extend(src2.reg, writable!(src2.reg), ext_kind)?;
+
+        context.stack.push(src2.into());
+        context.stack.push(src1.into());
+
+        self.v128_mul(context, kind.into())
+    }
+
+    fn v128_extadd_pairwise(
+        &mut self,
+        src: Reg,
+        dst: WritableReg,
+        kind: V128ExtAddKind,
+    ) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        match kind {
+            V128ExtAddKind::I8x16S => {
+                self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+                    // Use `vpmaddubsw` with a vector of 16 8-bit 1's which will
+                    // sign extend `src` to 16 bits and add adjacent words.
+                    // Need to supply constant as first operand since first operand
+                    // is treated as unsigned and the second operand is signed.
+                    let mask = masm.asm.add_constant(&[1; 16]);
+                    masm.asm.xmm_mov_mr(
+                        &mask,
+                        scratch.writable(),
+                        OperandSize::S128,
+                        MemFlags::trusted(),
+                    );
+                    masm.asm.xmm_vpmaddubsw_rrr(scratch.inner(), src, dst);
+                });
+            }
+            V128ExtAddKind::I8x16U => {
+                // Same approach as the signed variant but treat `src` as
+                // unsigned instead of signed by passing it as the first
+                // operand.
+                let mask = self.asm.add_constant(&[1; 16]);
+                self.asm.xmm_vpmaddubsw_rmr(src, &mask, dst);
+            }
+            V128ExtAddKind::I16x8S => {
+                // Similar approach to the two variants above. The vector is 8
+                // lanes of 16-bit 1's and `vpmaddwd` treats both operands as
+                // signed.
+                let mask = self
+                    .asm
+                    .add_constant(&[1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0]);
+                self.asm.xmm_vpmaddwd_rmr(src, &mask, dst);
+            }
+            V128ExtAddKind::I16x8U => {
+                // Similar approach as the signed variant.
+                // `vpmaddwd` operates on signed integers and the operand is
+                // unsigned so the operand needs to be converted to a signed
+                // format and than that process needs to be reversed after
+                // `vpmaddwd`.
+                // Flip the sign bit for 8 16-bit lanes.
+                let xor_mask = self.asm.add_constant(&[
+                    0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00,
+                    0x80, 0x00, 0x80,
+                ]);
+                self.asm.xmm_vpxor_rmr(src, &xor_mask, dst);
+
+                let madd_mask = self
+                    .asm
+                    .add_constant(&[1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0]);
+                self.asm.xmm_vpmaddwd_rmr(dst.to_reg(), &madd_mask, dst);
+
+                // Reverse the XOR. The XOR effectively subtracts 32,768 from
+                // both pairs that are added together so 65,536 (0x10000)
+                // needs to be added to 4 lanes of 32-bit values.
+                let add_mask = self
+                    .asm
+                    .add_constant(&[0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0]);
+                self.asm
+                    .xmm_vpadd_rmr(dst.to_reg(), &add_mask, dst, OperandSize::S32);
+            }
+        }
+        Ok(())
+    }
+
+    fn v128_dot(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vpmaddwd_rrr(lhs, rhs, dst);
+        Ok(())
+    }
+
+    fn v128_popcnt(&mut self, context: &mut CodeGenContext<Emission>) -> Result<()> {
+        self.ensure_has_avx()?;
+
+        let reg = writable!(context.pop_to_reg(self, None)?.reg);
+
+        // This works by using a lookup table to determine the count of bits
+        // set in the upper 4 bits and lower 4 bits separately and then adding
+        // the counts.
+
+        // A mask to zero out the upper 4 bits in each lane.
+        let address = self.asm.add_constant(&[
+            0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F,
+            0x0F, 0x0F,
+        ]);
+
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            // Zero out the upper 4 bits of each lane.
+            masm.asm
+                .xmm_vpand_rrm(reg.to_reg(), &address, scratch.writable());
+            // Right shift bytes in input by 4 bits to put the upper 4 bits in the
+            // lower 4 bits.
+            masm.asm
+                .xmm_vpsrl_rri(reg.to_reg(), reg, 0x4, OperandSize::S16);
+            // Zero out the upper 4 bits of each shifted lane.
+            masm.asm.xmm_vpand_rrm(reg.to_reg(), &address, reg);
+
+            // Write a lookup table of 4 bit values to number of bits set to a
+            // register so we only perform the memory read once.
+            // Index (hex) | Value (binary) | Population Count
+            // 0x0         | 0000          | 0
+            // 0x1         | 0001          | 1
+            // 0x2         | 0010          | 1
+            // 0x3         | 0011          | 2
+            // 0x4         | 0100          | 1
+            // 0x5         | 0101          | 2
+            // 0x6         | 0110          | 2
+            // 0x7         | 0111          | 3
+            // 0x8         | 1000          | 1
+            // 0x9         | 1001          | 2
+            // 0xA         | 1010          | 2
+            // 0xB         | 1011          | 3
+            // 0xC         | 1100          | 2
+            // 0xD         | 1101          | 3
+            // 0xE         | 1110          | 3
+            // 0xF         | 1111          | 4
+            let address = masm.asm.add_constant(&[
+                0x0, 0x1, 0x1, 0x2, 0x1, 0x2, 0x2, 0x3, 0x1, 0x2, 0x2, 0x3, 0x2, 0x3, 0x3, 0x4,
+            ]);
+            let reg2 = writable!(context.any_fpr(masm)?);
+            masm.asm
+                .xmm_mov_mr(&address, reg2, OperandSize::S128, MemFlags::trusted());
+            // Use the upper 4 bits as an index into the lookup table.
+            masm.asm.xmm_vpshufb_rrr(reg, reg2.to_reg(), reg.to_reg());
+            // Use the lower 4 bits as an index into the lookup table.
+            masm.asm
+                .xmm_vpshufb_rrr(scratch.writable(), reg2.to_reg(), scratch.inner());
+            context.free_reg(reg2.to_reg());
+
+            // Add the counts of the upper 4 bits and the lower 4 bits to get the
+            // total number of bits set.
+            masm.asm
+                .xmm_vpadd_rrr(reg.to_reg(), scratch.inner(), reg, OperandSize::S8);
+            anyhow::Ok(())
+        })?;
+
+        context.stack.push(TypedReg::v128(reg.to_reg()).into());
+        Ok(())
+    }
+
+    fn v128_avgr(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vpavg_rrr(lhs, rhs, dst, size);
+        Ok(())
+    }
+
+    fn v128_div(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vdivp_rrr(lhs, rhs, dst, size);
+        Ok(())
+    }
+
+    fn v128_sqrt(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm.xmm_vsqrtp_rr(src, dst, size);
+        Ok(())
+    }
+
+    fn v128_ceil(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm
+            .xmm_vroundp_rri(src, dst, VroundMode::TowardPositiveInfinity, size);
+        Ok(())
+    }
+
+    fn v128_floor(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm
+            .xmm_vroundp_rri(src, dst, VroundMode::TowardNegativeInfinity, size);
+        Ok(())
+    }
+
+    fn v128_nearest(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        self.asm
+            .xmm_vroundp_rri(src, dst, VroundMode::TowardNearest, size);
+        Ok(())
+    }
+
+    fn v128_pmin(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        // Reverse operands since Wasm specifies returning the first operand if
+        // either operand is NaN while x86 returns the second operand.
+        self.asm.xmm_vminp_rrr(rhs, lhs, dst, size);
+        Ok(())
+    }
+
+    fn v128_pmax(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+        self.ensure_has_avx()?;
+        // Reverse operands since Wasm specifies returning the first operand if
+        // either operand is NaN while x86 returns the second operand.
+        self.asm.xmm_vmaxp_rrr(rhs, lhs, dst, size);
+        Ok(())
+    }
+}
+
+impl MacroAssembler {
+    /// Create an x64 MacroAssembler.
+    pub fn new(
+        ptr_size: impl PtrSize,
+        shared_flags: settings::Flags,
+        isa_flags: x64_settings::Flags,
+    ) -> Result<Self> {
+        let ptr_type: WasmValType = ptr_type_from_ptr_size(ptr_size.size());
+
+        Ok(Self {
+            sp_offset: 0,
+            sp_max: 0,
+            stack_max_use_add: None,
+            asm: Assembler::new(shared_flags.clone(), isa_flags.clone()),
+            flags: isa_flags,
+            shared_flags,
+            ptr_size: ptr_type.try_into()?,
+            scratch_scope: RegAlloc::from(scratch_gpr_bitset(), scratch_fpr_bitset()),
+        })
+    }
+
+    /// Add the maximum stack used to a register, recording an obligation to update the
+    /// add-with-immediate instruction emitted to use the real stack max when the masm is being
+    /// finalized.
+    fn add_stack_max(&mut self, reg: Reg) {
+        assert!(self.stack_max_use_add.is_none());
+        let patch = PatchableAddToReg::new(reg, OperandSize::S64, &mut self.asm);
+        self.stack_max_use_add.replace(patch);
+    }
+
+    fn ensure_has_avx(&self) -> Result<()> {
+        anyhow::ensure!(self.flags.has_avx(), CodeGenError::UnimplementedForNoAvx);
+        Ok(())
+    }
+
+    fn ensure_has_avx2(&self) -> Result<()> {
+        anyhow::ensure!(self.flags.has_avx2(), CodeGenError::UnimplementedForNoAvx2);
+        Ok(())
+    }
+
+    fn ensure_has_avx512vl(&self) -> Result<()> {
+        anyhow::ensure!(
+            self.flags.has_avx512vl(),
+            CodeGenError::UnimplementedForNoAvx512VL
+        );
+        Ok(())
+    }
+
+    fn ensure_has_avx512dq(&self) -> Result<()> {
+        anyhow::ensure!(
+            self.flags.has_avx512dq(),
+            CodeGenError::UnimplementedForNoAvx512DQ
+        );
+        Ok(())
+    }
+
+    fn increment_sp(&mut self, bytes: u32) {
+        self.sp_offset += bytes;
+
+        // NOTE: we use `max` here to track the largest stack allocation in `sp_max`. Once we have
+        // seen the entire function, this value will represent the maximum size for the stack
+        // frame.
+        self.sp_max = self.sp_max.max(self.sp_offset);
+    }
+
+    fn decrement_sp(&mut self, bytes: u32) {
+        assert!(
+            self.sp_offset >= bytes,
+            "sp offset = {}; bytes = {}",
+            self.sp_offset,
+            bytes
+        );
+        self.sp_offset -= bytes;
+    }
+
+    fn load_constant(&mut self, constant: &I, dst: WritableReg, size: OperandSize) -> Result<()> {
+        match constant {
+            I::I32(v) => Ok(self.asm.mov_ir(*v as u64, dst, size)),
+            I::I64(v) => Ok(self.asm.mov_ir(*v, dst, size)),
+            I::F32(_) => Ok(self.asm.load_fp_const(dst, &constant.to_bytes(), size)),
+            I::F64(_) => Ok(self.asm.load_fp_const(dst, &constant.to_bytes(), size)),
+            I::V128(_) => Ok(self.asm.load_fp_const(dst, &constant.to_bytes(), size)),
+        }
+    }
+
+    /// A common implementation for zero-extend stack loads.
+    fn load_impl(
+        &mut self,
+        src: Address,
+        dst: WritableReg,
+        size: OperandSize,
+        flags: MemFlags,
+    ) -> Result<()> {
+        if dst.to_reg().is_int() {
+            let ext = size.extend_to::<Zero>(OperandSize::S64);
+            self.asm.movzx_mr(&src, dst, ext, flags);
+        } else {
+            self.asm.xmm_mov_mr(&src, dst, size, flags);
+        }
+
+        Ok(())
+    }
+
+    /// A common implementation for stack stores.
+    fn store_impl(
+        &mut self,
+        src: RegImm,
+        dst: Address,
+        size: OperandSize,
+        flags: MemFlags,
+    ) -> Result<()> {
+        let _ = match src {
+            RegImm::Imm(imm) => match imm {
+                I::I32(v) => self.asm.mov_im(v as i32, &dst, size, flags),
+                I::I64(v) => match v.try_into() {
+                    Ok(v) => self.asm.mov_im(v, &dst, size, flags),
+                    Err(_) => {
+                        // If the immediate doesn't sign extend, use a scratch
+                        // register.
+                        self.with_scratch::<IntScratch, _>(|masm, scratch| {
+                            masm.asm.mov_ir(v, scratch.writable(), size);
+                            masm.asm.mov_rm(scratch.inner(), &dst, size, flags);
+                        });
+                    }
+                },
+                I::F32(v) => {
+                    let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
+                    self.with_scratch::<FloatScratch, _>(|masm, float_scratch| {
+                        // Always trusted, since we are loading the constant from
+                        // the constant pool.
+                        masm.asm.xmm_mov_mr(
+                            &addr,
+                            float_scratch.writable(),
+                            size,
+                            MemFlags::trusted(),
+                        );
+                        masm.asm
+                            .xmm_mov_rm(float_scratch.inner(), &dst, size, flags);
+                    });
+                }
+                I::F64(v) => {
+                    let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
+
+                    self.with_scratch::<FloatScratch, _>(|masm, float_scratch| {
+                        // Similar to above, always trusted since we are loading the
+                        // constant from the constant pool.
+                        masm.asm.xmm_mov_mr(
+                            &addr,
+                            float_scratch.writable(),
+                            size,
+                            MemFlags::trusted(),
+                        );
+                        masm.asm
+                            .xmm_mov_rm(float_scratch.inner(), &dst, size, flags);
+                    });
+                }
+                I::V128(v) => {
+                    let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
+                    self.with_scratch::<FloatScratch, _>(|masm, vector_scratch| {
+                        // Always trusted, since we are loading the constant from
+                        // the constant pool.
+                        masm.asm.xmm_mov_mr(
+                            &addr,
+                            vector_scratch.writable(),
+                            size,
+                            MemFlags::trusted(),
+                        );
+                        masm.asm
+                            .xmm_mov_rm(vector_scratch.inner(), &dst, size, flags);
+                    });
+                }
+            },
+            RegImm::Reg(reg) => {
+                if reg.is_int() {
+                    self.asm.mov_rm(reg, &dst, size, flags);
+                } else {
+                    self.asm.xmm_mov_rm(reg, &dst, size, flags);
+                }
+            }
+        };
+        Ok(())
+    }
+
+    fn ensure_two_argument_form(dst: &Reg, lhs: &Reg) -> Result<()> {
+        if dst != lhs {
+            Err(anyhow!(CodeGenError::invalid_two_arg_form()))
+        } else {
+            Ok(())
+        }
+    }
+
+    /// The mask to use when performing a `vpshuf` operation for a 64-bit splat.
+    fn vpshuf_mask_for_64_bit_splats() -> u8 {
+        // Results in the first 4 bytes and second 4 bytes being
+        // swapped and then the swapped bytes being copied.
+        // [d0, d1, d2, d3, d4, d5, d6, d7, ...] yields
+        // [d4, d5, d6, d7, d0, d1, d2, d3, d4, d5, d6, d7, d0, d1, d2, d3].
+        0b01_00_01_00
+    }
+
+    fn v128_trunc_sat_f32x4_s(
+        &mut self,
+        reg: WritableReg,
+        src_lane_size: OperandSize,
+        dst_lane_size: OperandSize,
+    ) -> Result<()> {
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            // Create a mask to handle NaN values (1 for not NaN, 0 for
+            // NaN).
+            masm.asm.xmm_vcmpp_rrr(
+                scratch.writable(),
+                reg.to_reg(),
+                reg.to_reg(),
+                src_lane_size,
+                VcmpKind::Eq,
+            );
+            // Zero out any NaN values.
+            masm.asm
+                .xmm_vandp_rrr(reg.to_reg(), scratch.inner(), reg, src_lane_size);
+            // Create a mask for the sign bits.
+            masm.asm
+                .xmm_vpxor_rrr(scratch.inner(), reg.to_reg(), scratch.writable());
+            // Convert floats to integers.
+            masm.asm.xmm_vcvt_rr(reg.to_reg(), reg, VcvtKind::F32ToI32);
+            // Apply sign mask to the converted integers.
+            masm.asm
+                .xmm_vpand_rrr(reg.to_reg(), scratch.inner(), scratch.writable());
+            // Create a saturation mask of all 1s for negative numbers,
+            // all 0s for positive numbers. The arithmetic shift will cop
+            // the sign bit.
+            masm.asm
+                .xmm_vpsra_rri(scratch.inner(), scratch.writable(), 0x1F, dst_lane_size);
+            // Combine converted integers with saturation mask.
+            masm.asm.xmm_vpxor_rrr(reg.to_reg(), scratch.inner(), reg);
+            Ok(())
+        })
+    }
+
+    fn v128_trunc_sat_f32x4_u(
+        &mut self,
+        reg: WritableReg,
+        temp_reg: WritableReg,
+        src_lane_size: OperandSize,
+        dst_lane_size: OperandSize,
+    ) -> Result<()> {
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            // Set scratch to all zeros.
+            masm.asm.xmm_vxorp_rrr(
+                reg.to_reg(),
+                reg.to_reg(),
+                scratch.writable(),
+                src_lane_size,
+            );
+            // Clamp negative numbers to 0.
+            masm.asm
+                .xmm_vmaxp_rrr(reg.to_reg(), scratch.inner(), reg, src_lane_size);
+            // Create a vector of all 1s.
+            masm.asm.xmm_vpcmpeq_rrr(
+                scratch.writable(),
+                scratch.inner(),
+                scratch.inner(),
+                src_lane_size,
+            );
+            // Set scratch to 0x7FFFFFFF (max signed 32-bit integer) by
+            // performing a logical shift right.
+            masm.asm
+                .xmm_vpsrl_rri(scratch.inner(), scratch.writable(), 0x1, src_lane_size);
+            // Convert max signed int to float as a reference point for saturation.
+            masm.asm
+                .xmm_vcvt_rr(scratch.inner(), scratch.writable(), VcvtKind::I32ToF32);
+            // Convert the floats to integers and put the results in `reg2`.
+            // This is signed and not unsigned so we need to handle the
+            // value for the high bit in each lane.
+            masm.asm
+                .xmm_vcvt_rr(reg.to_reg(), temp_reg, VcvtKind::F32ToI32);
+            // Set `reg` lanes to the amount that the value in the lane
+            // exceeds the maximum signed 32-bit integer.
+            masm.asm
+                .xmm_vsub_rrr(reg.to_reg(), scratch.inner(), reg, dst_lane_size);
+            // Create mask in `scratch` for numbers that are larger than
+            // the maximum signed 32-bit integer. Lanes that don't fit
+            // in 32-bits ints will be 1.
+            masm.asm.xmm_vcmpp_rrr(
+                scratch.writable(),
+                scratch.inner(),
+                reg.to_reg(),
+                dst_lane_size,
+                VcmpKind::Le,
+            );
+            // Convert the excess over signed 32-bits from floats to integers.
+            masm.asm.xmm_vcvt_rr(reg.to_reg(), reg, VcvtKind::F32ToI32);
+            // Apply large number mask to excess values which will flip the
+            // bits in any lanes that exceed signed 32-bits. Adding this
+            // flipped value to the signed value will set the high bit and
+            // the carry behavior will update the other bits correctly.
+            masm.asm
+                .xmm_vpxor_rrr(reg.to_reg(), scratch.inner(), scratch.writable());
+            // Set `reg` to all 0s.
+            masm.asm.xmm_vpxor_rrr(reg.to_reg(), reg.to_reg(), reg);
+            // Ensure excess values are not negative by taking max b/w
+            // excess values and zero.
+            masm.asm
+                .xmm_vpmaxs_rrr(reg, scratch.inner(), reg.to_reg(), dst_lane_size);
+        });
+        // Perform the addition between the signed conversion value (in
+        // `reg2`) and the flipped excess value (in `reg`) to get the
+        // unsigned value.
+        self.asm
+            .xmm_vpadd_rrr(reg.to_reg(), temp_reg.to_reg(), reg, dst_lane_size);
+        Ok(())
+    }
+
+    fn v128_trunc_sat_f64x2_s_zero(
+        &mut self,
+        reg: WritableReg,
+        src_lane_size: OperandSize,
+    ) -> Result<()> {
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            // Create a NaN mask (1s for non-NaN, 0s for NaN).
+            masm.asm.xmm_vcmpp_rrr(
+                scratch.writable(),
+                reg.to_reg(),
+                reg.to_reg(),
+                src_lane_size,
+                VcmpKind::Eq,
+            );
+            // Clamp NaN values to maximum 64-bit float that can be
+            // converted to an i32.
+            let address = masm.asm.add_constant(&[
+                0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xDF, 0x41, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF,
+                0xDF, 0x41,
+            ]);
+            masm.asm
+                .xmm_vandp_rrm(scratch.inner(), &address, scratch.writable(), src_lane_size);
+            // Handle the saturation for values too large to fit in an i32.
+            masm.asm
+                .xmm_vminp_rrr(reg.to_reg(), scratch.inner(), reg, src_lane_size);
+            // Convert the floats to integers.
+            masm.asm.xmm_vcvt_rr(reg.to_reg(), reg, VcvtKind::F64ToI32);
+
+            Ok(())
+        })
+    }
+
+    fn v128_trunc_sat_f64x2_u_zero(
+        &mut self,
+        reg: WritableReg,
+        src_lane_size: OperandSize,
+        dst_lane_size: OperandSize,
+    ) -> Result<()> {
+        self.with_scratch::<FloatScratch, _>(|masm, scratch| {
+            // Zero out the scratch register.
+            masm.asm.xmm_vxorp_rrr(
+                scratch.inner(),
+                scratch.inner(),
+                scratch.writable(),
+                src_lane_size,
+            );
+            // Clamp negative values to zero.
+            masm.asm
+                .xmm_vmaxp_rrr(reg.to_reg(), scratch.inner(), reg, src_lane_size);
+            // Clamp value to maximum unsigned 32-bit integer value
+            // (0x41F0000000000000).
+            let address = masm.asm.add_constant(&[
+                0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xEF, 0x41, 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF,
+                0xEF, 0x41,
+            ]);
+            masm.asm
+                .xmm_vminp_rrm(reg.to_reg(), &address, reg, src_lane_size);
+            // Truncate floating point values.
+            masm.asm
+                .xmm_vroundp_rri(reg.to_reg(), reg, VroundMode::TowardZero, src_lane_size);
+            // Add 2^52 (doubles store 52 bits in their mantissa) to each
+            // lane causing values in the lower bits to be shifted into
+            // position for integer conversion.
+            let address = masm.asm.add_constant(&[
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x30, 0x43,
+            ]);
+            masm.asm
+                .xmm_vaddp_rrm(reg.to_reg(), &address, reg, src_lane_size);
+            // Takes lanes 0 and 2 from `reg` (converted values) and lanes
+            // 0 and 2 from `scratch` (zeroes) to put the converted ints in
+            // the lower lanes and zeroes in the upper lanes.
+            masm.asm.xmm_vshufp_rrri(
+                reg.to_reg(),
+                scratch.inner(),
+                reg,
+                0b10_00_10_00,
+                dst_lane_size,
+            );
+            Ok(())
+        })
+    }
+
+    /// Given a vector of floats where lanes with NaN values are set to all 1s
+    /// in `reg` and a vector register `dst` with a mix of non-NaN values and
+    /// possibly non-canonical NaN values, this canonicalize any NaNs in `dst`.
+    fn canonicalize_nans(&mut self, mask: WritableReg, dst: WritableReg, size: OperandSize) {
+        // Canonical NaNs do not preserve the sign bit, have the exponent bits
+        // all set, and have only the high bit of the mantissa set so shift by
+        // that number.
+        // The mask we're producing in this step will be inverted in the next
+        // step.
+        let amount_to_shift = 1 + size.mantissa_bits() + 1;
+        self.asm
+            .xmm_vpsrl_rri(mask.to_reg(), mask, amount_to_shift as u32, size);
+        // The mask will be inverted by the ANDN so non-NaN values will be all
+        // 1s and NaN values will set the sign bit, exponent bits, and zero out
+        // almost all of the mantissa.
+        self.asm
+            .xmm_vandnp_rrr(mask.to_reg(), dst.to_reg(), dst, size);
+    }
+}