wasmtime 37.0.2 → 38.0.4

data/ext/cargo-vendor/cranelift-codegen-0.125.4/src/isa/x64/inst/emit.rs

@@ -0,0 +1,2188 @@
+use crate::ir::KnownSymbol;
+use crate::ir::immediates::{Ieee32, Ieee64};
+use crate::isa::x64::external::{AsmInst, CraneliftRegisters, PairedGpr};
+use crate::isa::x64::inst::args::*;
+use crate::isa::x64::inst::*;
+use crate::isa::x64::lower::isle::generated_code::{Atomic128RmwSeqOp, AtomicRmwSeqOp};
+use cranelift_assembler_x64 as asm;
+
+/// A small helper to generate a signed conversion instruction.
+fn emit_signed_cvt(
+    sink: &mut MachBuffer<Inst>,
+    info: &EmitInfo,
+    state: &mut EmitState,
+    src: Reg,
+    dst: Writable<Reg>,
+    to_f64: bool,
+) {
+    assert!(src.is_real());
+    assert!(dst.to_reg().is_real());
+
+    // Handle an unsigned int, which is the "easy" case: a signed conversion
+    // will do the right thing.
+    let dst = WritableXmm::from_writable_reg(dst).unwrap();
+    if to_f64 {
+        asm::inst::cvtsi2sdq_a::new(dst, src).emit(sink, info, state);
+    } else {
+        asm::inst::cvtsi2ssq_a::new(dst, src).emit(sink, info, state);
+    }
+}
+
+/// Emits a one way conditional jump if CC is set (true).
+fn one_way_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
+    let cond_start = sink.cur_offset();
+    let cond_disp_off = cond_start + 2;
+    sink.use_label_at_offset(cond_disp_off, label, LabelUse::JmpRel32);
+    emit_jcc_no_offset(sink, cc);
+    debug_assert_eq!(sink.cur_offset(), cond_disp_off + 4);
+}
+
+/// Like `one_way_jmp` above emitting a conditional jump, but also using
+/// `MachBuffer::add_cond_branch`.
+fn cond_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
+    let cond_start = sink.cur_offset();
+    let cond_disp_off = cond_start + 2;
+    let cond_end = cond_start + 6;
+
+    sink.use_label_at_offset(cond_disp_off, label, LabelUse::JmpRel32);
+    // FIXME: ideally this `inverted` calculation would go through the external
+    // assembler, but for now it's left done manually.
+    let inverted: [u8; 6] = [0x0F, 0x80 + (cc.invert().get_enc()), 0x00, 0x00, 0x00, 0x00];
+    sink.add_cond_branch(cond_start, cond_end, label, &inverted[..]);
+
+    emit_jcc_no_offset(sink, cc);
+
+    debug_assert_eq!(sink.cur_offset(), cond_disp_off + 4);
+    debug_assert_eq!(sink.cur_offset(), cond_end);
+}
+
+fn emit_jcc_no_offset(sink: &mut MachBuffer<Inst>, cc: CC) {
+    // Note that the disassembler matches Capstone which doesn't match the `CC`
+    // enum directly as Intel has multiple mnemonics use the same encoding.
+    let inst: AsmInst = match cc {
+        CC::Z => asm::inst::je_d32::new(0).into(),   // jz == je
+        CC::NZ => asm::inst::jne_d32::new(0).into(), // jnz == jne
+        CC::B => asm::inst::jb_d32::new(0).into(),
+        CC::NB => asm::inst::jae_d32::new(0).into(), // jnb == jae
+        CC::BE => asm::inst::jbe_d32::new(0).into(),
+        CC::NBE => asm::inst::ja_d32::new(0).into(), // jnbe == ja
+        CC::L => asm::inst::jl_d32::new(0).into(),
+        CC::LE => asm::inst::jle_d32::new(0).into(),
+        CC::NL => asm::inst::jge_d32::new(0).into(), // jnl == jge
+        CC::NLE => asm::inst::jg_d32::new(0).into(), // jnle == jg
+        CC::O => asm::inst::jo_d32::new(0).into(),
+        CC::NO => asm::inst::jno_d32::new(0).into(),
+        CC::P => asm::inst::jp_d32::new(0).into(),
+        CC::NP => asm::inst::jnp_d32::new(0).into(),
+        CC::S => asm::inst::js_d32::new(0).into(),
+        CC::NS => asm::inst::jns_d32::new(0).into(),
+    };
+    inst.encode(&mut external::AsmCodeSink {
+        sink,
+        incoming_arg_offset: 0,
+        slot_offset: 0,
+    });
+}
+
+/// Emits an unconditional branch.
+fn uncond_jmp(sink: &mut MachBuffer<Inst>, label: MachLabel) {
+    let uncond_start = sink.cur_offset();
+    let uncond_disp_off = uncond_start + 1;
+    let uncond_end = uncond_start + 5;
+
+    sink.use_label_at_offset(uncond_disp_off, label, LabelUse::JmpRel32);
+    sink.add_uncond_branch(uncond_start, uncond_end, label);
+
+    asm::inst::jmp_d32::new(0).encode(&mut external::AsmCodeSink {
+        sink,
+        incoming_arg_offset: 0,
+        slot_offset: 0,
+    });
+    debug_assert_eq!(sink.cur_offset(), uncond_disp_off + 4);
+    debug_assert_eq!(sink.cur_offset(), uncond_end);
+}
+
+/// Emits a relocation, attaching the current source location as well.
+fn emit_reloc(sink: &mut MachBuffer<Inst>, kind: Reloc, name: &ExternalName, addend: Addend) {
+    sink.add_reloc(kind, name, addend);
+}
+
+/// The top-level emit function.
+///
+/// Important!  Do not add improved (shortened) encoding cases to existing
+/// instructions without also adding tests for those improved encodings.  That
+/// is a dangerous game that leads to hard-to-track-down errors in the emitted
+/// code.
+///
+/// For all instructions, make sure to have test coverage for all of the
+/// following situations.  Do this by creating the cross product resulting from
+/// applying the following rules to each operand:
+///
+/// (1) for any insn that mentions a register: one test using a register from
+///     the group [rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi] and a second one
+///     using a register from the group [r8, r9, r10, r11, r12, r13, r14, r15].
+///     This helps detect incorrect REX prefix construction.
+///
+/// (2) for any insn that mentions a byte register: one test for each of the
+///     four encoding groups [al, cl, dl, bl], [spl, bpl, sil, dil],
+///     [r8b .. r11b] and [r12b .. r15b].  This checks that
+///     apparently-redundant REX prefixes are retained when required.
+///
+/// (3) for any insn that contains an immediate field, check the following
+///     cases: field is zero, field is in simm8 range (-128 .. 127), field is
+///     in simm32 range (-0x8000_0000 .. 0x7FFF_FFFF).  This is because some
+///     instructions that require a 32-bit immediate have a short-form encoding
+///     when the imm is in simm8 range.
+///
+/// Rules (1), (2) and (3) don't apply for registers within address expressions
+/// (`Addr`s).  Those are already pretty well tested, and the registers in them
+/// don't have any effect on the containing instruction (apart from possibly
+/// require REX prefix bits).
+///
+/// When choosing registers for a test, avoid using registers with the same
+/// offset within a given group.  For example, don't use rax and r8, since they
+/// both have the lowest 3 bits as 000, and so the test won't detect errors
+/// where those 3-bit register sub-fields are confused by the emitter.  Instead
+/// use (eg) rax (lo3 = 000) and r9 (lo3 = 001).  Similarly, don't use (eg) cl
+/// and bpl since they have the same offset in their group; use instead (eg) cl
+/// and sil.
+///
+/// For all instructions, also add a test that uses only low-half registers
+/// (rax .. rdi, xmm0 .. xmm7) etc, so as to check that any redundant REX
+/// prefixes are correctly omitted.  This low-half restriction must apply to
+/// _all_ registers in the insn, even those in address expressions.
+///
+/// Following these rules creates large numbers of test cases, but it's the
+/// only way to make the emitter reliable.
+///
+/// Known possible improvements:
+///
+/// * there's a shorter encoding for shl/shr/sar by a 1-bit immediate.  (Do we
+///   care?)
+pub(crate) fn emit(
+    inst: &Inst,
+    sink: &mut MachBuffer<Inst>,
+    info: &EmitInfo,
+    state: &mut EmitState,
+) {
+    if !inst.is_available(&info) {
+        let features = if let Inst::External { inst } = inst {
+            inst.features().to_string()
+        } else {
+            "see `is_available` source for feature term".to_string()
+        };
+        panic!(
+            "Cannot emit inst '{inst:?}' for target; failed to match ISA requirements: {features}"
+        );
+    }
+
+    match inst {
+        Inst::CheckedSRemSeq { divisor, .. } | Inst::CheckedSRemSeq8 { divisor, .. } => {
+            // Validate that the register constraints of the dividend and the
+            // destination are all as expected.
+            let (dst, size) = match inst {
+                Inst::CheckedSRemSeq {
+                    dividend_lo,
+                    dividend_hi,
+                    dst_quotient,
+                    dst_remainder,
+                    size,
+                    ..
+                } => {
+                    let dividend_lo = dividend_lo.to_reg();
+                    let dividend_hi = dividend_hi.to_reg();
+                    let dst_quotient = dst_quotient.to_reg().to_reg();
+                    let dst_remainder = dst_remainder.to_reg().to_reg();
+                    debug_assert_eq!(dividend_lo, regs::rax());
+                    debug_assert_eq!(dividend_hi, regs::rdx());
+                    debug_assert_eq!(dst_quotient, regs::rax());
+                    debug_assert_eq!(dst_remainder, regs::rdx());
+                    (regs::rdx(), *size)
+                }
+                Inst::CheckedSRemSeq8 { dividend, dst, .. } => {
+                    let dividend = dividend.to_reg();
+                    let dst = dst.to_reg().to_reg();
+                    debug_assert_eq!(dividend, regs::rax());
+                    debug_assert_eq!(dst, regs::rax());
+                    (regs::rax(), OperandSize::Size8)
+                }
+                _ => unreachable!(),
+            };
+
+            // Generates the following code sequence:
+            //
+            // cmp -1 %divisor
+            // jnz $do_op
+            //
+            // ;; for srem, result is 0
+            // mov #0, %dst
+            // j $done
+            //
+            // $do_op:
+            // idiv %divisor
+            //
+            // $done:
+
+            let do_op = sink.get_label();
+            let done_label = sink.get_label();
+
+            // Check if the divisor is -1, and if it isn't then immediately
+            // go to the `idiv`.
+            let inst = Inst::cmp_mi_sxb(size, *divisor, -1);
+            inst.emit(sink, info, state);
+            one_way_jmp(sink, CC::NZ, do_op);
+
+            // ... otherwise the divisor is -1 and the result is always 0. This
+            // is written to the destination register which will be %rax for
+            // 8-bit srem and %rdx otherwise.
+            //
+            // Note that for 16-to-64-bit srem operations this leaves the
+            // second destination, %rax, unchanged. This isn't semantically
+            // correct if a lowering actually tries to use the `dst_quotient`
+            // output but for srem only the `dst_remainder` output is used for
+            // now.
+            let inst = Inst::imm(OperandSize::Size64, 0, Writable::from_reg(dst));
+            inst.emit(sink, info, state);
+            let inst = Inst::jmp_known(done_label);
+            inst.emit(sink, info, state);
+
+            // Here the `idiv` is executed, which is different depending on the
+            // size
+            sink.bind_label(do_op, state.ctrl_plane_mut());
+            let rax = Gpr::RAX;
+            let rdx = Gpr::RDX;
+            let writable_rax = Writable::from_reg(rax);
+            let writable_rdx = Writable::from_reg(rdx);
+            let inst: AsmInst = match size {
+                OperandSize::Size8 => asm::inst::idivb_m::new(
+                    PairedGpr::from(writable_rax),
+                    *divisor,
+                    TrapCode::INTEGER_DIVISION_BY_ZERO,
+                )
+                .into(),
+
+                OperandSize::Size16 => asm::inst::idivw_m::new(
+                    PairedGpr::from(writable_rax),
+                    PairedGpr::from(writable_rdx),
+                    *divisor,
+                    TrapCode::INTEGER_DIVISION_BY_ZERO,
+                )
+                .into(),
+
+                OperandSize::Size32 => asm::inst::idivl_m::new(
+                    PairedGpr::from(writable_rax),
+                    PairedGpr::from(writable_rdx),
+                    *divisor,
+                    TrapCode::INTEGER_DIVISION_BY_ZERO,
+                )
+                .into(),
+
+                OperandSize::Size64 => asm::inst::idivq_m::new(
+                    PairedGpr::from(writable_rax),
+                    PairedGpr::from(writable_rdx),
+                    *divisor,
+                    TrapCode::INTEGER_DIVISION_BY_ZERO,
+                )
+                .into(),
+            };
+            inst.emit(sink, info, state);
+
+            sink.bind_label(done_label, state.ctrl_plane_mut());
+        }
+
+        Inst::MovFromPReg { src, dst } => {
+            let src: Reg = (*src).into();
+            debug_assert!([regs::rsp(), regs::rbp(), regs::pinned_reg()].contains(&src));
+            asm::inst::movq_mr::new(*dst, Gpr::unwrap_new(src)).emit(sink, info, state);
+        }
+
+        Inst::MovToPReg { src, dst } => {
+            let dst: Reg = (*dst).into();
+            debug_assert!([regs::rsp(), regs::rbp(), regs::pinned_reg()].contains(&dst));
+            let dst = WritableGpr::from_writable_reg(Writable::from_reg(dst)).unwrap();
+            asm::inst::movq_mr::new(dst, *src).emit(sink, info, state);
+        }
+
+        Inst::XmmCmove {
+            ty,
+            cc,
+            consequent,
+            alternative,
+            dst,
+        } => {
+            let alternative = *alternative;
+            let dst = *dst;
+            debug_assert_eq!(alternative, dst.to_reg());
+            let consequent = *consequent;
+
+            // Lowering of the Select IR opcode when the input is an fcmp relies on the fact that
+            // this doesn't clobber flags. Make sure to not do so here.
+            let next = sink.get_label();
+
+            // Jump if cc is *not* set.
+            one_way_jmp(sink, cc.invert(), next);
+            Inst::gen_move(dst.map(|r| r.to_reg()), consequent.to_reg(), *ty)
+                .emit(sink, info, state);
+
+            sink.bind_label(next, state.ctrl_plane_mut());
+        }
+
+        Inst::StackProbeLoop {
+            tmp,
+            frame_size,
+            guard_size,
+        } => {
+            assert!(info.flags.enable_probestack());
+            assert!(guard_size.is_power_of_two());
+
+            let tmp = *tmp;
+
+            // Number of probes that we need to perform
+            let probe_count = align_to(*frame_size, *guard_size) / guard_size;
+
+            // The inline stack probe loop has 3 phases:
+            //
+            // We generate the "guard area" register which is essentially the frame_size aligned to
+            // guard_size. We copy the stack pointer and subtract the guard area from it. This
+            // gets us a register that we can use to compare when looping.
+            //
+            // After that we emit the loop. Essentially we just adjust the stack pointer one guard_size'd
+            // distance at a time and then touch the stack by writing anything to it. We use the previously
+            // created "guard area" register to know when to stop looping.
+            //
+            // When we have touched all the pages that we need, we have to restore the stack pointer
+            // to where it was before.
+            //
+            // Generate the following code:
+            //         mov  tmp_reg, rsp
+            //         sub  tmp_reg, guard_size * probe_count
+            // .loop_start:
+            //         sub  rsp, guard_size
+            //         mov  [rsp], rsp
+            //         cmp  rsp, tmp_reg
+            //         jne  .loop_start
+            //         add  rsp, guard_size * probe_count
+
+            // Create the guard bound register
+            // mov  tmp_reg, rsp
+            let inst = Inst::gen_move(tmp, regs::rsp(), types::I64);
+            inst.emit(sink, info, state);
+
+            // sub  tmp_reg, GUARD_SIZE * probe_count
+            let guard_plus_count = i32::try_from(guard_size * probe_count)
+                .expect("`guard_size * probe_count` is too large to fit in a 32-bit immediate");
+            Inst::subq_mi(tmp, guard_plus_count).emit(sink, info, state);
+
+            // Emit the main loop!
+            let loop_start = sink.get_label();
+            sink.bind_label(loop_start, state.ctrl_plane_mut());
+
+            // sub  rsp, GUARD_SIZE
+            let rsp = Writable::from_reg(regs::rsp());
+            let guard_size_ = i32::try_from(*guard_size)
+                .expect("`guard_size` is too large to fit in a 32-bit immediate");
+            Inst::subq_mi(rsp, guard_size_).emit(sink, info, state);
+
+            // TODO: `mov [rsp], 0` would be better, but we don't have that instruction
+            // Probe the stack! We don't use Inst::gen_store_stack here because we need a predictable
+            // instruction size.
+            // mov  [rsp], rsp
+            asm::inst::movl_mr::new(Amode::imm_reg(0, regs::rsp()), Gpr::RSP)
+                .emit(sink, info, state);
+
+            // Compare and jump if we are not done yet
+            // cmp  rsp, tmp_reg
+            let tmp = Gpr::unwrap_new(tmp.to_reg());
+            asm::inst::cmpq_rm::new(tmp, Gpr::RSP).emit(sink, info, state);
+
+            // jne  .loop_start
+            // TODO: Encoding the conditional jump as a short jump
+            // could save us us 4 bytes here.
+            one_way_jmp(sink, CC::NZ, loop_start);
+
+            // The regular prologue code is going to emit a `sub` after this, so we need to
+            // reset the stack pointer
+            //
+            // TODO: It would be better if we could avoid the `add` + `sub` that is generated here
+            // and in the stack adj portion of the prologue
+            //
+            // add rsp, GUARD_SIZE * probe_count
+            Inst::addq_mi(rsp, guard_plus_count).emit(sink, info, state);
+        }
+
+        Inst::CallKnown { info: call_info } => {
+            let stack_map = state.take_stack_map();
+
+            asm::inst::callq_d::new(0).emit(sink, info, state);
+
+            // The last 4 bytes of `callq` is the relative displacement to where
+            // we're calling, so that's where the reloc is registered.
+            //
+            // The addend adjusts for the difference between the end of the
+            // instruction and the beginning of the immediate field.
+            let len = sink.cur_offset();
+            sink.add_reloc_at_offset(len - 4, Reloc::X86CallPCRel4, &call_info.dest, -4);
+
+            if let Some(s) = stack_map {
+                sink.push_user_stack_map(state, len, s);
+            }
+
+            if let Some(try_call) = call_info.try_call_info.as_ref() {
+                sink.add_try_call_site(
+                    Some(state.frame_layout().sp_to_fp()),
+                    try_call.exception_handlers(&state.frame_layout()),
+                );
+            } else {
+                sink.add_call_site();
+            }
+
+            // Reclaim the outgoing argument area that was released by the
+            // callee, to ensure that StackAMode values are always computed from
+            // a consistent SP.
+            if call_info.callee_pop_size > 0 {
+                let rsp = Writable::from_reg(regs::rsp());
+                let callee_pop_size = i32::try_from(call_info.callee_pop_size)
+                    .expect("`callee_pop_size` is too large to fit in a 32-bit immediate");
+                Inst::subq_mi(rsp, callee_pop_size).emit(sink, info, state);
+            }
+
+            // Load any stack-carried return values.
+            call_info.emit_retval_loads::<X64ABIMachineSpec, _, _>(
+                state.frame_layout().stackslots_size,
+                |inst| inst.emit(sink, info, state),
+                |_space_needed| None,
+            );
+
+            // If this is a try-call, jump to the continuation
+            // (normal-return) block.
+            if let Some(try_call) = call_info.try_call_info.as_ref() {
+                let jmp = Inst::JmpKnown {
+                    dst: try_call.continuation,
+                };
+                jmp.emit(sink, info, state);
+            }
+        }
+
+        Inst::ReturnCallKnown { info: call_info } => {
+            emit_return_call_common_sequence(sink, info, state, &call_info);
+
+            // Finally, jump to the callee!
+            //
+            // Note: this is not `Inst::Jmp { .. }.emit(..)` because we have
+            // different metadata in this case: we don't have a label for the
+            // target, but rather a function relocation.
+            asm::inst::jmp_d32::new(0).emit(sink, info, state);
+            let offset = sink.cur_offset();
+            // The addend adjusts for the difference between the end of the instruction and the
+            // beginning of the immediate field.
+            sink.add_reloc_at_offset(offset - 4, Reloc::X86CallPCRel4, &call_info.dest, -4);
+            sink.add_call_site();
+        }
+
+        Inst::ReturnCallUnknown { info: call_info } => {
+            let callee = call_info.dest;
+
+            emit_return_call_common_sequence(sink, info, state, &call_info);
+
+            asm::inst::jmpq_m::new(callee).emit(sink, info, state);
+            sink.add_call_site();
+        }
+
+        Inst::CallUnknown {
+            info: call_info, ..
+        } => {
+            let stack_map = state.take_stack_map();
+
+            let dest = match call_info.dest.clone() {
+                RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::unwrap_new(reg)),
+                RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
+            };
+
+            asm::inst::callq_m::new(dest).emit(sink, info, state);
+
+            if let Some(s) = stack_map {
+                let offset = sink.cur_offset();
+                sink.push_user_stack_map(state, offset, s);
+            }
+
+            if let Some(try_call) = call_info.try_call_info.as_ref() {
+                sink.add_try_call_site(
+                    Some(state.frame_layout().sp_to_fp()),
+                    try_call.exception_handlers(&state.frame_layout()),
+                );
+            } else {
+                sink.add_call_site();
+            }
+
+            // Reclaim the outgoing argument area that was released by the callee, to ensure that
+            // StackAMode values are always computed from a consistent SP.
+            if call_info.callee_pop_size > 0 {
+                let rsp = Writable::from_reg(regs::rsp());
+                let callee_pop_size = i32::try_from(call_info.callee_pop_size)
+                    .expect("`callee_pop_size` is too large to fit in a 32-bit immediate");
+                Inst::subq_mi(rsp, callee_pop_size).emit(sink, info, state);
+            }
+
+            // Load any stack-carried return values.
+            call_info.emit_retval_loads::<X64ABIMachineSpec, _, _>(
+                state.frame_layout().stackslots_size,
+                |inst| inst.emit(sink, info, state),
+                |_space_needed| None,
+            );
+
+            if let Some(try_call) = call_info.try_call_info.as_ref() {
+                let jmp = Inst::JmpKnown {
+                    dst: try_call.continuation,
+                };
+                jmp.emit(sink, info, state);
+            }
+        }
+
+        Inst::Args { .. } => {}
+        Inst::Rets { .. } => {}
+
+        Inst::StackSwitchBasic {
+            store_context_ptr,
+            load_context_ptr,
+            in_payload0,
+            out_payload0,
+        } => {
+            // Note that we do not emit anything for preserving and restoring
+            // ordinary registers here: That's taken care of by regalloc for us,
+            // since we marked this instruction as clobbering all registers.
+            //
+            // Also note that we do nothing about passing the single payload
+            // value: We've informed regalloc that it is sent and received via
+            // the fixed register given by [stack_switch::payload_register]
+
+            let (tmp1, tmp2) = {
+                // Ideally we would just ask regalloc for two temporary registers.
+                // However, adding any early defs to the constraints on StackSwitch
+                // causes TooManyLiveRegs. Fortunately, we can manually find tmp
+                // registers without regalloc: Since our instruction clobbers all
+                // registers, we can simply pick any register that is not assigned
+                // to the operands.
+
+                let all = crate::isa::x64::abi::ALL_CLOBBERS;
+
+                let used_regs = [
+                    **load_context_ptr,
+                    **store_context_ptr,
+                    **in_payload0,
+                    *out_payload0.to_reg(),
+                ];
+
+                let mut tmps = all.into_iter().filter_map(|preg| {
+                    let reg: Reg = preg.into();
+                    if !used_regs.contains(&reg) {
+                        WritableGpr::from_writable_reg(isle::WritableReg::from_reg(reg))
+                    } else {
+                        None
+                    }
+                });
+                (tmps.next().unwrap(), tmps.next().unwrap())
+            };
+
+            let layout = stack_switch::control_context_layout();
+            let rsp_offset = layout.stack_pointer_offset as i32;
+            let pc_offset = layout.ip_offset as i32;
+            let rbp_offset = layout.frame_pointer_offset as i32;
+
+            // Location to which someone switch-ing back to this stack will jump
+            // to: Right behind the `StackSwitch` instruction
+            let resume = sink.get_label();
+
+            //
+            // For RBP and RSP we do the following:
+            // - Load new value for register from `load_context_ptr` +
+            // corresponding offset.
+            // - Store previous (!) value of register at `store_context_ptr` +
+            // corresponding offset.
+            //
+            // Since `load_context_ptr` and `store_context_ptr` are allowed to be
+            // equal, we need to use a temporary register here.
+            //
+
+            let mut exchange = |offset, reg| {
+                let addr = SyntheticAmode::real(Amode::imm_reg(offset, **load_context_ptr));
+                asm::inst::movq_rm::new(tmp1, addr).emit(sink, info, state);
+
+                asm::inst::movq_mr::new(
+                    Amode::imm_reg(offset, **store_context_ptr),
+                    Gpr::new(reg).unwrap(),
+                )
+                .emit(sink, info, state);
+
+                let dst = Writable::from_reg(reg);
+                asm::inst::movq_mr::new(dst.map(Gpr::unwrap_new), tmp1.to_reg())
+                    .emit(sink, info, state);
+            };
+
+            exchange(rsp_offset, regs::rsp());
+            exchange(rbp_offset, regs::rbp());
+
+            //
+            // Load target PC, store resume PC, jump to target PC
+            //
+
+            let addr = SyntheticAmode::real(Amode::imm_reg(pc_offset, **load_context_ptr));
+            asm::inst::movq_rm::new(tmp1, addr).emit(sink, info, state);
+
+            let amode = Amode::RipRelative { target: resume };
+            asm::inst::leaq_rm::new(tmp2, amode).emit(sink, info, state);
+
+            asm::inst::movq_mr::new(
+                Amode::imm_reg(pc_offset, **store_context_ptr),
+                tmp2.to_reg(),
+            )
+            .emit(sink, info, state);
+
+            asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state);
+
+            sink.bind_label(resume, state.ctrl_plane_mut());
+        }
+
+        Inst::JmpKnown { dst } => uncond_jmp(sink, *dst),
+
+        Inst::WinchJmpIf { cc, taken } => one_way_jmp(sink, *cc, *taken),
+
+        Inst::JmpCond {
+            cc,
+            taken,
+            not_taken,
+        } => {
+            cond_jmp(sink, *cc, *taken);
+            uncond_jmp(sink, *not_taken);
+        }
+
+        Inst::JmpCondOr {
+            cc1,
+            cc2,
+            taken,
+            not_taken,
+        } => {
+            // Emit:
+            //   jcc1 taken
+            //   jcc2 taken
+            //   jmp not_taken
+            //
+            // Note that we enroll both conditionals in the
+            // branch-chomping mechanism because MachBuffer
+            // simplification can continue upward as long as it keeps
+            // chomping branches. In the best case, if taken ==
+            // not_taken and that one block is the fallthrough block,
+            // all three branches can disappear.
+
+            cond_jmp(sink, *cc1, *taken);
+            cond_jmp(sink, *cc2, *taken);
+            uncond_jmp(sink, *not_taken);
+        }
+
+        &Inst::JmpTableSeq {
+            idx,
+            tmp1,
+            tmp2,
+            ref targets,
+            ref default_target,
+            ..
+        } => {
+            // This sequence is *one* instruction in the vcode, and is expanded only here at
+            // emission time, because we cannot allow the regalloc to insert spills/reloads in
+            // the middle; we depend on hardcoded PC-rel addressing below.
+            //
+            // We don't have to worry about emitting islands, because the only label-use type has a
+            // maximum range of 2 GB. If we later consider using shorter-range label references,
+            // this will need to be revisited.
+
+            // We generate the following sequence. Note that the only read of %idx is before the
+            // write to %tmp2, so regalloc may use the same register for both; fix x64/inst/mod.rs
+            // if you change this.
+            // lea start_of_jump_table_offset(%rip), %tmp1
+            // movslq [%tmp1, %idx, 4], %tmp2 ;; shift of 2, viz. multiply index by 4
+            // addq %tmp2, %tmp1
+            // j *%tmp1
+            // $start_of_jump_table:
+            // -- jump table entries
+
+            // Load base address of jump table.
+            let start_of_jumptable = sink.get_label();
+            asm::inst::leaq_rm::new(tmp1, Amode::rip_relative(start_of_jumptable))
+                .emit(sink, info, state);
+
+            // Load value out of the jump table. It's a relative offset to the target block, so it
+            // might be negative; use a sign-extension.
+            let inst = Inst::movsx_rm_r(
+                ExtMode::LQ,
+                RegMem::mem(Amode::imm_reg_reg_shift(
+                    0,
+                    Gpr::unwrap_new(tmp1.to_reg()),
+                    Gpr::unwrap_new(idx),
+                    2,
+                )),
+                tmp2,
+            );
+            inst.emit(sink, info, state);
+
+            // Add base of jump table to jump-table-sourced block offset.
+            asm::inst::addq_rm::new(tmp1, tmp2).emit(sink, info, state);
+
+            // Branch to computed address.
+            asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state);
+
+            // Emit jump table (table of 32-bit offsets).
+            sink.bind_label(start_of_jumptable, state.ctrl_plane_mut());
+            let jt_off = sink.cur_offset();
+            for &target in targets.iter().chain(std::iter::once(default_target)) {
+                let word_off = sink.cur_offset();
+                // off_into_table is an addend here embedded in the label to be later patched at
+                // the end of codegen. The offset is initially relative to this jump table entry;
+                // with the extra addend, it'll be relative to the jump table's start, after
+                // patching.
+                let off_into_table = word_off - jt_off;
+                sink.use_label_at_offset(word_off, target, LabelUse::PCRel32);
+                sink.put4(off_into_table);
+            }
+        }
+
+        Inst::TrapIf { cc, trap_code } => {
+            let trap_label = sink.defer_trap(*trap_code);
+            one_way_jmp(sink, *cc, trap_label);
+        }
+
+        Inst::TrapIfAnd {
+            cc1,
+            cc2,
+            trap_code,
+        } => {
+            let trap_label = sink.defer_trap(*trap_code);
+            let else_label = sink.get_label();
+
+            // Jump to the end if the first condition isn't true, and then if
+            // the second condition is true go to the trap.
+            one_way_jmp(sink, cc1.invert(), else_label);
+            one_way_jmp(sink, *cc2, trap_label);
+
+            sink.bind_label(else_label, state.ctrl_plane_mut());
+        }
+
+        Inst::TrapIfOr {
+            cc1,
+            cc2,
+            trap_code,
+        } => {
+            let trap_label = sink.defer_trap(*trap_code);
+
+            // Emit two jumps to the same trap if either condition code is true.
+            one_way_jmp(sink, *cc1, trap_label);
+            one_way_jmp(sink, *cc2, trap_label);
+        }
+
+        Inst::XmmMinMaxSeq {
+            size,
+            is_min,
+            lhs,
+            rhs,
+            dst,
+        } => {
+            let rhs = rhs.to_reg();
+            let lhs = lhs.to_reg();
+            let dst = dst.to_writable_reg();
+            debug_assert_eq!(rhs, dst.to_reg());
+
+            // Generates the following sequence:
+            // cmpss/cmpsd %lhs, %rhs_dst
+            // jnz do_min_max
+            // jp propagate_nan
+            //
+            // ;; ordered and equal: propagate the sign bit (for -0 vs 0):
+            // {and,or}{ss,sd} %lhs, %rhs_dst
+            // j done
+            //
+            // ;; to get the desired NaN behavior (signalling NaN transformed into a quiet NaN, the
+            // ;; NaN value is returned), we add both inputs.
+            // propagate_nan:
+            // add{ss,sd} %lhs, %rhs_dst
+            // j done
+            //
+            // do_min_max:
+            // {min,max}{ss,sd} %lhs, %rhs_dst
+            //
+            // done:
+            let done = sink.get_label();
+            let propagate_nan = sink.get_label();
+            let do_min_max = sink.get_label();
+
+            let (add_op, cmp_op, and_op, or_op, min_max_op) = match size {
+                OperandSize::Size32 => (
+                    asm::inst::addss_a::new(dst, lhs).into(),
+                    asm::inst::ucomiss_a::new(dst.to_reg(), lhs).into(),
+                    asm::inst::andps_a::new(dst, lhs).into(),
+                    asm::inst::orps_a::new(dst, lhs).into(),
+                    if *is_min {
+                        asm::inst::minss_a::new(dst, lhs).into()
+                    } else {
+                        asm::inst::maxss_a::new(dst, lhs).into()
+                    },
+                ),
+                OperandSize::Size64 => (
+                    asm::inst::addsd_a::new(dst, lhs).into(),
+                    asm::inst::ucomisd_a::new(dst.to_reg(), lhs).into(),
+                    asm::inst::andpd_a::new(dst, lhs).into(),
+                    asm::inst::orpd_a::new(dst, lhs).into(),
+                    if *is_min {
+                        asm::inst::minsd_a::new(dst, lhs).into()
+                    } else {
+                        asm::inst::maxsd_a::new(dst, lhs).into()
+                    },
+                ),
+                _ => unreachable!(),
+            };
+            let add_op: AsmInst = add_op;
+            let or_op: AsmInst = or_op;
+            let min_max_op: AsmInst = min_max_op;
+            let cmp_op: AsmInst = cmp_op;
+
+            cmp_op.emit(sink, info, state);
+
+            one_way_jmp(sink, CC::NZ, do_min_max);
+            one_way_jmp(sink, CC::P, propagate_nan);
+
+            // Ordered and equal. The operands are bit-identical unless they are zero
+            // and negative zero. These instructions merge the sign bits in that
+            // case, and are no-ops otherwise.
+            let inst: AsmInst = if *is_min { or_op } else { and_op };
+            inst.emit(sink, info, state);
+
+            let inst = Inst::jmp_known(done);
+            inst.emit(sink, info, state);
+
+            // x86's min/max are not symmetric; if either operand is a NaN, they return the
+            // read-only operand: perform an addition between the two operands, which has the
+            // desired NaN propagation effects.
+            sink.bind_label(propagate_nan, state.ctrl_plane_mut());
+            add_op.emit(sink, info, state);
+
+            one_way_jmp(sink, CC::P, done);
+
+            sink.bind_label(do_min_max, state.ctrl_plane_mut());
+            min_max_op.emit(sink, info, state);
+
+            sink.bind_label(done, state.ctrl_plane_mut());
+        }
+
+        Inst::XmmUninitializedValue { .. } | Inst::GprUninitializedValue { .. } => {
+            // These instruction formats only exist to declare a register as a
+            // `def`; no code is emitted. This is always immediately followed by
+            // an instruction, such as `xor <tmp>, <tmp>`, that semantically
+            // reads this undefined value but arithmetically produces the same
+            // result regardless of its value.
+        }
+
+        Inst::CvtUint64ToFloatSeq {
+            dst_size,
+            src,
+            dst,
+            tmp_gpr1,
+            tmp_gpr2,
+        } => {
+            let src = src.to_reg();
+            let dst = dst.to_writable_reg();
+            let tmp_gpr1 = tmp_gpr1.to_writable_reg();
+            let tmp_gpr2 = tmp_gpr2.to_writable_reg();
+
+            // Note: this sequence is specific to 64-bit mode; a 32-bit mode would require a
+            // different sequence.
+            //
+            // Emit the following sequence:
+            //
+            //  cmp 0, %src
+            //  jl handle_negative
+            //
+            //  ;; handle positive, which can't overflow
+            //  cvtsi2sd/cvtsi2ss %src, %dst
+            //  j done
+            //
+            //  ;; handle negative: see below for an explanation of what it's doing.
+            //  handle_negative:
+            //  mov %src, %tmp_gpr1
+            //  shr $1, %tmp_gpr1
+            //  mov %src, %tmp_gpr2
+            //  and $1, %tmp_gpr2
+            //  or %tmp_gpr1, %tmp_gpr2
+            //  cvtsi2sd/cvtsi2ss %tmp_gpr2, %dst
+            //  addsd/addss %dst, %dst
+            //
+            //  done:
+
+            assert_ne!(src, tmp_gpr1.to_reg());
+            assert_ne!(src, tmp_gpr2.to_reg());
+
+            let handle_negative = sink.get_label();
+            let done = sink.get_label();
+
+            // If x seen as a signed int64 is not negative, a signed-conversion will do the right
+            // thing.
+            // TODO use tst src, src here.
+            asm::inst::cmpq_mi_sxb::new(src, 0).emit(sink, info, state);
+
+            one_way_jmp(sink, CC::L, handle_negative);
+
+            // Handle a positive int64, which is the "easy" case: a signed conversion will do the
+            // right thing.
+            emit_signed_cvt(
+                sink,
+                info,
+                state,
+                src,
+                dst,
+                *dst_size == OperandSize::Size64,
+            );
+
+            let inst = Inst::jmp_known(done);
+            inst.emit(sink, info, state);
+
+            sink.bind_label(handle_negative, state.ctrl_plane_mut());
+
+            // Divide x by two to get it in range for the signed conversion, keep the LSB, and
+            // scale it back up on the FP side.
+            let inst = Inst::gen_move(tmp_gpr1, src, types::I64);
+            inst.emit(sink, info, state);
+
+            // tmp_gpr1 := src >> 1
+            asm::inst::shrq_mi::new(tmp_gpr1, 1).emit(sink, info, state);
+
+            let inst = Inst::gen_move(tmp_gpr2, src, types::I64);
+            inst.emit(sink, info, state);
+
+            asm::inst::andq_mi_sxb::new(tmp_gpr2, 1).emit(sink, info, state);
+
+            asm::inst::orq_rm::new(tmp_gpr2, tmp_gpr1).emit(sink, info, state);
+
+            emit_signed_cvt(
+                sink,
+                info,
+                state,
+                tmp_gpr2.to_reg(),
+                dst,
+                *dst_size == OperandSize::Size64,
+            );
+
+            let inst: AsmInst = match *dst_size {
+                OperandSize::Size64 => asm::inst::addsd_a::new(dst, dst.to_reg()).into(),
+                OperandSize::Size32 => asm::inst::addss_a::new(dst, dst.to_reg()).into(),
+                _ => unreachable!(),
+            };
+            inst.emit(sink, info, state);
+
+            sink.bind_label(done, state.ctrl_plane_mut());
+        }
+
+        Inst::CvtFloatToSintSeq {
+            src_size,
+            dst_size,
+            is_saturating,
+            src,
+            dst,
+            tmp_gpr,
+            tmp_xmm,
+        } => {
+            use OperandSize::*;
+
+            let src = src.to_reg();
+            let dst = dst.to_writable_reg();
+            let tmp_gpr = tmp_gpr.to_writable_reg();
+            let tmp_xmm = tmp_xmm.to_writable_reg();
+
+            // Emits the following common sequence:
+            //
+            // cvttss2si/cvttsd2si %src, %dst
+            // cmp %dst, 1
+            // jno done
+            //
+            // Then, for saturating conversions:
+            //
+            // ;; check for NaN
+            // cmpss/cmpsd %src, %src
+            // jnp not_nan
+            // xor %dst, %dst
+            //
+            // ;; positive inputs get saturated to INT_MAX; negative ones to INT_MIN, which is
+            // ;; already in %dst.
+            // xorpd %tmp_xmm, %tmp_xmm
+            // cmpss/cmpsd %src, %tmp_xmm
+            // jnb done
+            // mov/movaps $INT_MAX, %dst
+            //
+            // done:
+            //
+            // Then, for non-saturating conversions:
+            //
+            // ;; check for NaN
+            // cmpss/cmpsd %src, %src
+            // jnp not_nan
+            // ud2 trap BadConversionToInteger
+            //
+            // ;; check if INT_MIN was the correct result, against a magic constant:
+            // not_nan:
+            // movaps/mov $magic, %tmp_gpr
+            // movq/movd %tmp_gpr, %tmp_xmm
+            // cmpss/cmpsd %tmp_xmm, %src
+            // jnb/jnbe $check_positive
+            // ud2 trap IntegerOverflow
+            //
+            // ;; if positive, it was a real overflow
+            // check_positive:
+            // xorpd %tmp_xmm, %tmp_xmm
+            // cmpss/cmpsd %src, %tmp_xmm
+            // jnb done
+            // ud2 trap IntegerOverflow
+            //
+            // done:
+
+            let cmp_op: AsmInst = match src_size {
+                Size64 => asm::inst::ucomisd_a::new(src, src).into(),
+                Size32 => asm::inst::ucomiss_a::new(src, src).into(),
+                _ => unreachable!(),
+            };
+
+            let cvtt_op = |dst, src| Inst::External {
+                inst: match (*src_size, *dst_size) {
+                    (Size32, Size32) => asm::inst::cvttss2si_a::new(dst, src).into(),
+                    (Size32, Size64) => asm::inst::cvttss2si_aq::new(dst, src).into(),
+                    (Size64, Size32) => asm::inst::cvttsd2si_a::new(dst, src).into(),
+                    (Size64, Size64) => asm::inst::cvttsd2si_aq::new(dst, src).into(),
+                    _ => unreachable!(),
+                },
+            };
+
+            let done = sink.get_label();
+
+            // The truncation.
+            cvtt_op(dst, src).emit(sink, info, state);
+
+            // Compare against 1, in case of overflow the dst operand was INT_MIN.
+            let inst = Inst::cmp_mi_sxb(*dst_size, Gpr::unwrap_new(dst.to_reg()), 1);
+            inst.emit(sink, info, state);
+
+            one_way_jmp(sink, CC::NO, done); // no overflow => done
+
+            // Check for NaN.
+            cmp_op.emit(sink, info, state);
+
+            if *is_saturating {
+                let not_nan = sink.get_label();
+                one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN
+
+                // For NaN, emit 0.
+                let inst: AsmInst = match *dst_size {
+                    OperandSize::Size32 => asm::inst::xorl_rm::new(dst, dst).into(),
+                    OperandSize::Size64 => asm::inst::xorq_rm::new(dst, dst).into(),
+                    _ => unreachable!(),
+                };
+                inst.emit(sink, info, state);
+
+                let inst = Inst::jmp_known(done);
+                inst.emit(sink, info, state);
+
+                sink.bind_label(not_nan, state.ctrl_plane_mut());
+
+                // If the input was positive, saturate to INT_MAX.
+
+                // Zero out tmp_xmm.
+                asm::inst::xorpd_a::new(tmp_xmm, tmp_xmm.to_reg()).emit(sink, info, state);
+
+                let inst: AsmInst = match src_size {
+                    Size64 => asm::inst::ucomisd_a::new(tmp_xmm.to_reg(), src).into(),
+                    Size32 => asm::inst::ucomiss_a::new(tmp_xmm.to_reg(), src).into(),
+                    _ => unreachable!(),
+                };
+                inst.emit(sink, info, state);
+
+                // Jump if >= to done.
+                one_way_jmp(sink, CC::NB, done);
+
+                // Otherwise, put INT_MAX.
+                if *dst_size == OperandSize::Size64 {
+                    let inst = Inst::imm(OperandSize::Size64, 0x7fffffffffffffff, dst);
+                    inst.emit(sink, info, state);
+                } else {
+                    let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, dst);
+                    inst.emit(sink, info, state);
+                }
+            } else {
+                let inst = Inst::trap_if(CC::P, TrapCode::BAD_CONVERSION_TO_INTEGER);
+                inst.emit(sink, info, state);
+
+                // Check if INT_MIN was the correct result: determine the smallest floating point
+                // number that would convert to INT_MIN, put it in a temporary register, and compare
+                // against the src register.
+                // If the src register is less (or in some cases, less-or-equal) than the threshold,
+                // trap!
+
+                let mut no_overflow_cc = CC::NB; // >=
+                let output_bits = dst_size.to_bits();
+                match *src_size {
+                    OperandSize::Size32 => {
+                        let cst = (-Ieee32::pow2(output_bits - 1)).bits();
+                        let inst = Inst::imm(OperandSize::Size32, cst as u64, tmp_gpr);
+                        inst.emit(sink, info, state);
+                    }
+                    OperandSize::Size64 => {
+                        // An f64 can represent `i32::min_value() - 1` exactly with precision to spare,
+                        // so there are values less than -2^(N-1) that convert correctly to INT_MIN.
+                        let cst = if output_bits < 64 {
+                            no_overflow_cc = CC::NBE; // >
+                            Ieee64::fcvt_to_sint_negative_overflow(output_bits)
+                        } else {
+                            -Ieee64::pow2(output_bits - 1)
+                        };
+                        let inst = Inst::imm(OperandSize::Size64, cst.bits(), tmp_gpr);
+                        inst.emit(sink, info, state);
+                    }
+                    _ => unreachable!(),
+                }
+
+                let inst: AsmInst = {
+                    let tmp_xmm: WritableXmm = tmp_xmm.map(|r| Xmm::new(r).unwrap());
+                    match src_size {
+                        Size32 => asm::inst::movd_a::new(tmp_xmm, tmp_gpr).into(),
+                        Size64 => asm::inst::movq_a::new(tmp_xmm, tmp_gpr).into(),
+                        _ => unreachable!(),
+                    }
+                };
+                inst.emit(sink, info, state);
+
+                let inst: AsmInst = match src_size {
+                    Size64 => asm::inst::ucomisd_a::new(src, tmp_xmm.to_reg()).into(),
+                    Size32 => asm::inst::ucomiss_a::new(src, tmp_xmm.to_reg()).into(),
+                    _ => unreachable!(),
+                };
+                inst.emit(sink, info, state);
+
+                // no trap if src >= or > threshold
+                let inst = Inst::trap_if(no_overflow_cc.invert(), TrapCode::INTEGER_OVERFLOW);
+                inst.emit(sink, info, state);
+
+                // If positive, it was a real overflow.
+
+                // Zero out the tmp_xmm register.
+                asm::inst::xorpd_a::new(tmp_xmm, tmp_xmm.to_reg()).emit(sink, info, state);
+
+                let inst: AsmInst = match src_size {
+                    Size64 => asm::inst::ucomisd_a::new(tmp_xmm.to_reg(), src).into(),
+                    Size32 => asm::inst::ucomiss_a::new(tmp_xmm.to_reg(), src).into(),
+                    _ => unreachable!(),
+                };
+                inst.emit(sink, info, state);
+
+                // no trap if 0 >= src
+                let inst = Inst::trap_if(CC::B, TrapCode::INTEGER_OVERFLOW);
+                inst.emit(sink, info, state);
+            }
+
+            sink.bind_label(done, state.ctrl_plane_mut());
+        }
+
+        Inst::CvtFloatToUintSeq {
+            src_size,
+            dst_size,
+            is_saturating,
+            src,
+            dst,
+            tmp_gpr,
+            tmp_xmm,
+            tmp_xmm2,
+        } => {
+            use OperandSize::*;
+
+            let src = src.to_reg();
+            let dst = dst.to_writable_reg();
+            let tmp_gpr = tmp_gpr.to_writable_reg();
+            let tmp_xmm = tmp_xmm.to_writable_reg();
+            let tmp_xmm2 = tmp_xmm2.to_writable_reg();
+
+            // The only difference in behavior between saturating and non-saturating is how we
+            // handle errors. Emits the following sequence:
+            //
+            // movaps/mov 2**(int_width - 1), %tmp_gpr
+            // movq/movd %tmp_gpr, %tmp_xmm
+            // cmpss/cmpsd %tmp_xmm, %src
+            // jnb is_large
+            //
+            // ;; check for NaN inputs
+            // jnp not_nan
+            // -- non-saturating: ud2 trap BadConversionToInteger
+            // -- saturating: xor %dst, %dst; j done
+            //
+            // not_nan:
+            // cvttss2si/cvttsd2si %src, %dst
+            // cmp 0, %dst
+            // jnl done
+            // -- non-saturating: ud2 trap IntegerOverflow
+            // -- saturating: xor %dst, %dst; j done
+            //
+            // is_large:
+            // mov %src, %tmp_xmm2
+            // subss/subsd %tmp_xmm, %tmp_xmm2
+            // cvttss2si/cvttss2sd %tmp_x, %dst
+            // cmp 0, %dst
+            // jnl next_is_large
+            // -- non-saturating: ud2 trap IntegerOverflow
+            // -- saturating: movaps $UINT_MAX, %dst; j done
+            //
+            // next_is_large:
+            // add 2**(int_width -1), %dst ;; 2 instructions for 64-bits integers
+            //
+            // done:
+
+            assert_ne!(tmp_xmm.to_reg(), src, "tmp_xmm clobbers src!");
+
+            let xor_op = |dst, src| Inst::External {
+                inst: match *dst_size {
+                    Size32 => asm::inst::xorl_rm::new(dst, src).into(),
+                    Size64 => asm::inst::xorq_rm::new(dst, src).into(),
+                    _ => unreachable!(),
+                },
+            };
+
+            let subs_op = |dst, src| Inst::External {
+                inst: match *src_size {
+                    Size32 => asm::inst::subss_a::new(dst, src).into(),
+                    Size64 => asm::inst::subsd_a::new(dst, src).into(),
+                    _ => unreachable!(),
+                },
+            };
+
+            let cvtt_op = |dst, src| Inst::External {
+                inst: match (*src_size, *dst_size) {
+                    (Size32, Size32) => asm::inst::cvttss2si_a::new(dst, src).into(),
+                    (Size32, Size64) => asm::inst::cvttss2si_aq::new(dst, src).into(),
+                    (Size64, Size32) => asm::inst::cvttsd2si_a::new(dst, src).into(),
+                    (Size64, Size64) => asm::inst::cvttsd2si_aq::new(dst, src).into(),
+                    _ => unreachable!(),
+                },
+            };
+
+            let done = sink.get_label();
+
+            let cst = match src_size {
+                OperandSize::Size32 => Ieee32::pow2(dst_size.to_bits() - 1).bits() as u64,
+                OperandSize::Size64 => Ieee64::pow2(dst_size.to_bits() - 1).bits(),
+                _ => unreachable!(),
+            };
+
+            let inst = Inst::imm(*src_size, cst, tmp_gpr);
+            inst.emit(sink, info, state);
+
+            let inst: AsmInst = {
+                let tmp_xmm: WritableXmm = tmp_xmm.map(|r| Xmm::new(r).unwrap());
+                match src_size {
+                    Size32 => asm::inst::movd_a::new(tmp_xmm, tmp_gpr).into(),
+                    Size64 => asm::inst::movq_a::new(tmp_xmm, tmp_gpr).into(),
+                    _ => unreachable!(),
+                }
+            };
+            inst.emit(sink, info, state);
+
+            let inst: AsmInst = match src_size {
+                Size64 => asm::inst::ucomisd_a::new(src, tmp_xmm.to_reg()).into(),
+                Size32 => asm::inst::ucomiss_a::new(src, tmp_xmm.to_reg()).into(),
+                _ => unreachable!(),
+            };
+            inst.emit(sink, info, state);
+
+            let handle_large = sink.get_label();
+            one_way_jmp(sink, CC::NB, handle_large); // jump to handle_large if src >= large_threshold
+
+            if *is_saturating {
+                // If not NaN jump over this 0-return, otherwise return 0
+                let not_nan = sink.get_label();
+                one_way_jmp(sink, CC::NP, not_nan);
+
+                xor_op(dst, dst).emit(sink, info, state);
+
+                let inst = Inst::jmp_known(done);
+                inst.emit(sink, info, state);
+                sink.bind_label(not_nan, state.ctrl_plane_mut());
+            } else {
+                // Trap.
+                let inst = Inst::trap_if(CC::P, TrapCode::BAD_CONVERSION_TO_INTEGER);
+                inst.emit(sink, info, state);
+            }
+
+            // Actual truncation for small inputs: if the result is not positive, then we had an
+            // overflow.
+
+            cvtt_op(dst, src).emit(sink, info, state);
+
+            let inst = Inst::cmp_mi_sxb(*dst_size, Gpr::unwrap_new(dst.to_reg()), 0);
+            inst.emit(sink, info, state);
+
+            one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done
+
+            if *is_saturating {
+                // The input was "small" (< 2**(width -1)), so the only way to get an integer
+                // overflow is because the input was too small: saturate to the min value, i.e. 0.
+                let inst: AsmInst = match *dst_size {
+                    OperandSize::Size32 => asm::inst::xorl_rm::new(dst, dst).into(),
+                    OperandSize::Size64 => asm::inst::xorq_rm::new(dst, dst).into(),
+                    _ => unreachable!(),
+                };
+                inst.emit(sink, info, state);
+
+                let inst = Inst::jmp_known(done);
+                inst.emit(sink, info, state);
+            } else {
+                // Trap.
+                asm::inst::ud2_zo::new(TrapCode::INTEGER_OVERFLOW).emit(sink, info, state);
+            }
+
+            // Now handle large inputs.
+
+            sink.bind_label(handle_large, state.ctrl_plane_mut());
+
+            let inst = Inst::gen_move(tmp_xmm2, src, types::F64);
+            inst.emit(sink, info, state);
+
+            subs_op(tmp_xmm2, tmp_xmm.to_reg()).emit(sink, info, state);
+
+            cvtt_op(dst, tmp_xmm2.to_reg()).emit(sink, info, state);
+
+            let inst = Inst::cmp_mi_sxb(*dst_size, Gpr::unwrap_new(dst.to_reg()), 0);
+            inst.emit(sink, info, state);
+
+            if *is_saturating {
+                let next_is_large = sink.get_label();
+                one_way_jmp(sink, CC::NL, next_is_large); // if dst >= 0, jump to next_is_large
+
+                // The input was "large" (>= 2**(width -1)), so the only way to get an integer
+                // overflow is because the input was too large: saturate to the max value.
+                let inst = Inst::imm(
+                    OperandSize::Size64,
+                    if *dst_size == OperandSize::Size64 {
+                        u64::max_value()
+                    } else {
+                        u32::max_value() as u64
+                    },
+                    dst,
+                );
+                inst.emit(sink, info, state);
+
+                let inst = Inst::jmp_known(done);
+                inst.emit(sink, info, state);
+                sink.bind_label(next_is_large, state.ctrl_plane_mut());
+            } else {
+                let inst = Inst::trap_if(CC::L, TrapCode::INTEGER_OVERFLOW);
+                inst.emit(sink, info, state);
+            }
+
+            if *dst_size == OperandSize::Size64 {
+                let inst = Inst::imm(OperandSize::Size64, 1 << 63, tmp_gpr);
+                inst.emit(sink, info, state);
+
+                asm::inst::addq_rm::new(dst, tmp_gpr).emit(sink, info, state);
+            } else {
+                asm::inst::addl_mi::new(dst, asm::Imm32::new(1 << 31)).emit(sink, info, state);
+            }
+
+            sink.bind_label(done, state.ctrl_plane_mut());
+        }
+
+        Inst::LoadExtName {
+            dst,
+            name,
+            offset,
+            distance,
+        } => {
+            let name = &**name;
+            let riprel = asm::Amode::RipRelative {
+                target: asm::DeferredTarget::None,
+            };
+            if info.flags.is_pic() {
+                // Generates: movq symbol@GOTPCREL(%rip), %dst
+                asm::inst::movq_rm::new(*dst, riprel).emit(sink, info, state);
+                let cur = sink.cur_offset();
+                sink.add_reloc_at_offset(cur - 4, Reloc::X86GOTPCRel4, name, -4);
+
+                // Offset in the relocation above applies to the address of the
+                // *GOT entry*, not the loaded address; so we emit a separate
+                // add instruction if needed.
+                let offset = i32::try_from(*offset).unwrap();
+                if offset != 0 {
+                    asm::inst::addq_mi_sxl::new(PairedGpr::from(*dst), offset)
+                        .emit(sink, info, state);
+                }
+            } else if distance == &RelocDistance::Near {
+                // If we know the distance to the name is within 2GB (e.g., a
+                // module-local function), we can generate a RIP-relative
+                // address, with a relocation.
+                asm::inst::leaq_rm::new(*dst, riprel).emit(sink, info, state);
+                let cur = sink.cur_offset();
+                sink.add_reloc_at_offset(cur - 4, Reloc::X86CallPCRel4, name, *offset - 4);
+            } else {
+                // The full address can be encoded in the register, with a
+                // relocation.
+                asm::inst::movabsq_oi::new(*dst, 0).emit(sink, info, state);
+                let cur = sink.cur_offset();
+                sink.add_reloc_at_offset(cur - 8, Reloc::Abs8, name, *offset);
+            }
+        }
+
+        Inst::AtomicRmwSeq {
+            ty,
+            op,
+            mem,
+            operand,
+            temp,
+            dst_old,
+        } => {
+            let operand = *operand;
+            let temp = *temp;
+            let temp_r = temp.map(|r| *r);
+            let dst_old = *dst_old;
+            let dst_old_r = dst_old.map(|r| *r);
+            debug_assert_eq!(dst_old.to_reg(), regs::rax());
+            let mem = mem.finalize(state.frame_layout(), sink).clone();
+
+            // Emit this:
+            //    mov{zbq,zwq,zlq,q}     (%r_address), %rax    // rax = old value
+            //  again:
+            //    movq                   %rax, %r_temp         // rax = old value, r_temp = old value
+            //    `op`q                  %r_operand, %r_temp   // rax = old value, r_temp = new value
+            //    lock cmpxchg{b,w,l,q}  %r_temp, (%r_address) // try to store new value
+            //    jnz again // If this is taken, rax will have a "revised" old value
+            //
+            // Operand conventions: IN:  %r_address, %r_operand OUT: %rax (old
+            //    value), %r_temp (trashed), %rflags (trashed)
+            let again_label = sink.get_label();
+
+            // mov{zbq,zwq,zlq,q} (%r_address), %rax
+            // No need to call `add_trap` here, since the `i1` emit will do that.
+            let i1 = Inst::load(*ty, mem.clone(), dst_old_r, ExtKind::ZeroExtend);
+            i1.emit(sink, info, state);
+
+            // again:
+            sink.bind_label(again_label, state.ctrl_plane_mut());
+
+            // movq %rax, %r_temp
+            asm::inst::movq_mr::new(temp, dst_old.to_reg()).emit(sink, info, state);
+
+            use AtomicRmwSeqOp as RmwOp;
+            match op {
+                RmwOp::Nand => {
+                    // andq %r_operand, %r_temp
+                    asm::inst::andq_rm::new(temp, operand).emit(sink, info, state);
+
+                    // notq %r_temp
+                    asm::inst::notq_m::new(PairedGpr::from(temp)).emit(sink, info, state);
+                }
+                RmwOp::Umin | RmwOp::Umax | RmwOp::Smin | RmwOp::Smax => {
+                    // cmp %r_temp, %r_operand
+                    let temp = temp.to_reg();
+                    match *ty {
+                        types::I8 => asm::inst::cmpb_mr::new(operand, temp).emit(sink, info, state),
+                        types::I16 => {
+                            asm::inst::cmpw_mr::new(operand, temp).emit(sink, info, state)
+                        }
+                        types::I32 => {
+                            asm::inst::cmpl_mr::new(operand, temp).emit(sink, info, state)
+                        }
+                        types::I64 => {
+                            asm::inst::cmpq_mr::new(operand, temp).emit(sink, info, state)
+                        }
+                        _ => unreachable!(),
+                    }
+
+                    // cmovcc %r_operand, %r_temp
+                    match op {
+                        RmwOp::Umin => {
+                            asm::inst::cmovbeq_rm::new(temp_r, *operand).emit(sink, info, state)
+                        }
+                        RmwOp::Umax => {
+                            asm::inst::cmovaeq_rm::new(temp_r, *operand).emit(sink, info, state)
+                        }
+                        RmwOp::Smin => {
+                            asm::inst::cmovleq_rm::new(temp_r, *operand).emit(sink, info, state)
+                        }
+                        RmwOp::Smax => {
+                            asm::inst::cmovgeq_rm::new(temp_r, *operand).emit(sink, info, state)
+                        }
+                        _ => unreachable!(),
+                    }
+                }
+                RmwOp::And => {
+                    // andq %r_operand, %r_temp
+                    asm::inst::andq_rm::new(temp, operand).emit(sink, info, state);
+                }
+                RmwOp::Or => {
+                    // orq %r_operand, %r_temp
+                    asm::inst::orq_rm::new(temp, operand).emit(sink, info, state);
+                }
+                RmwOp::Xor => {
+                    // xorq %r_operand, %r_temp
+                    asm::inst::xorq_rm::new(temp, operand).emit(sink, info, state);
+                }
+            }
+
+            // lock cmpxchg{b,w,l,q} %r_temp, (%r_address)
+            // No need to call `add_trap` here, since the `i4` emit will do that.
+            let temp = temp.to_reg();
+            let dst_old = PairedGpr::from(dst_old);
+            let inst: AsmInst = match *ty {
+                types::I8 => asm::inst::lock_cmpxchgb_mr::new(mem, temp, dst_old).into(),
+                types::I16 => asm::inst::lock_cmpxchgw_mr::new(mem, temp, dst_old).into(),
+                types::I32 => asm::inst::lock_cmpxchgl_mr::new(mem, temp, dst_old).into(),
+                types::I64 => asm::inst::lock_cmpxchgq_mr::new(mem, temp, dst_old).into(),
+                _ => unreachable!(),
+            };
+            inst.emit(sink, info, state);
+
+            // jnz again
+            one_way_jmp(sink, CC::NZ, again_label);
+        }
+
+        Inst::Atomic128RmwSeq {
+            op,
+            mem,
+            operand_low,
+            operand_high,
+            temp_low,
+            temp_high,
+            dst_old_low,
+            dst_old_high,
+        } => {
+            let operand_low = *operand_low;
+            let operand_high = *operand_high;
+            let temp_low = *temp_low;
+            let temp_high = *temp_high;
+            let dst_old_low = *dst_old_low;
+            let dst_old_high = *dst_old_high;
+            debug_assert_eq!(temp_low.to_reg(), regs::rbx());
+            debug_assert_eq!(temp_high.to_reg(), regs::rcx());
+            debug_assert_eq!(dst_old_low.to_reg(), regs::rax());
+            debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());
+            let mem = mem.finalize(state.frame_layout(), sink).clone();
+
+            let again_label = sink.get_label();
+
+            // Load the initial value.
+            asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);
+            asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);
+
+            // again:
+            sink.bind_label(again_label, state.ctrl_plane_mut());
+
+            // Move old value to temp registers.
+            asm::inst::movq_mr::new(temp_low, dst_old_low.to_reg()).emit(sink, info, state);
+            asm::inst::movq_mr::new(temp_high, dst_old_high.to_reg()).emit(sink, info, state);
+
+            // Perform the operation.
+            use Atomic128RmwSeqOp as RmwOp;
+            match op {
+                RmwOp::Nand => {
+                    // temp &= operand
+                    asm::inst::andq_rm::new(temp_low, operand_low).emit(sink, info, state);
+                    asm::inst::andq_rm::new(temp_high, operand_high).emit(sink, info, state);
+
+                    // temp = !temp
+                    asm::inst::notq_m::new(PairedGpr::from(temp_low)).emit(sink, info, state);
+                    asm::inst::notq_m::new(PairedGpr::from(temp_high)).emit(sink, info, state);
+                }
+                RmwOp::Umin | RmwOp::Umax | RmwOp::Smin | RmwOp::Smax => {
+                    // Do a comparison with LHS temp and RHS operand.
+                    // Note the opposite argument orders.
+                    asm::inst::cmpq_mr::new(temp_low.to_reg(), operand_low).emit(sink, info, state);
+                    // This will clobber `temp_high`
+                    asm::inst::sbbq_rm::new(temp_high, operand_high).emit(sink, info, state);
+                    // Restore the clobbered value
+                    asm::inst::movq_mr::new(temp_high, dst_old_high.to_reg())
+                        .emit(sink, info, state);
+                    match op {
+                        RmwOp::Umin => {
+                            asm::inst::cmovaeq_rm::new(temp_low, operand_low)
+                                .emit(sink, info, state);
+                            asm::inst::cmovaeq_rm::new(temp_high, operand_high)
+                                .emit(sink, info, state);
+                        }
+                        RmwOp::Umax => {
+                            asm::inst::cmovbq_rm::new(temp_low, operand_low)
+                                .emit(sink, info, state);
+                            asm::inst::cmovbq_rm::new(temp_high, operand_high)
+                                .emit(sink, info, state);
+                        }
+                        RmwOp::Smin => {
+                            asm::inst::cmovgeq_rm::new(temp_low, operand_low)
+                                .emit(sink, info, state);
+                            asm::inst::cmovgeq_rm::new(temp_high, operand_high)
+                                .emit(sink, info, state);
+                        }
+                        RmwOp::Smax => {
+                            asm::inst::cmovlq_rm::new(temp_low, operand_low)
+                                .emit(sink, info, state);
+                            asm::inst::cmovlq_rm::new(temp_high, operand_high)
+                                .emit(sink, info, state);
+                        }
+                        _ => unreachable!(),
+                    }
+                }
+                RmwOp::Add => {
+                    asm::inst::addq_rm::new(temp_low, operand_low).emit(sink, info, state);
+                    asm::inst::adcq_rm::new(temp_high, operand_high).emit(sink, info, state);
+                }
+                RmwOp::Sub => {
+                    asm::inst::subq_rm::new(temp_low, operand_low).emit(sink, info, state);
+                    asm::inst::sbbq_rm::new(temp_high, operand_high).emit(sink, info, state);
+                }
+                RmwOp::And => {
+                    asm::inst::andq_rm::new(temp_low, operand_low).emit(sink, info, state);
+                    asm::inst::andq_rm::new(temp_high, operand_high).emit(sink, info, state);
+                }
+                RmwOp::Or => {
+                    asm::inst::orq_rm::new(temp_low, operand_low).emit(sink, info, state);
+                    asm::inst::orq_rm::new(temp_high, operand_high).emit(sink, info, state);
+                }
+                RmwOp::Xor => {
+                    asm::inst::xorq_rm::new(temp_low, operand_low).emit(sink, info, state);
+                    asm::inst::xorq_rm::new(temp_high, operand_high).emit(sink, info, state);
+                }
+            }
+
+            // cmpxchg16b (mem)
+            asm::inst::lock_cmpxchg16b_m::new(
+                PairedGpr::from(dst_old_low),
+                PairedGpr::from(dst_old_high),
+                temp_low.to_reg(),
+                temp_high.to_reg(),
+                mem,
+            )
+            .emit(sink, info, state);
+
+            // jnz again
+            one_way_jmp(sink, CC::NZ, again_label);
+        }
+
+        Inst::Atomic128XchgSeq {
+            mem,
+            operand_low,
+            operand_high,
+            dst_old_low,
+            dst_old_high,
+        } => {
+            let operand_low = *operand_low;
+            let operand_high = *operand_high;
+            let dst_old_low = *dst_old_low;
+            let dst_old_high = *dst_old_high;
+            debug_assert_eq!(operand_low, regs::rbx());
+            debug_assert_eq!(operand_high, regs::rcx());
+            debug_assert_eq!(dst_old_low.to_reg(), regs::rax());
+            debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());
+            let mem = mem.finalize(state.frame_layout(), sink).clone();
+
+            let again_label = sink.get_label();
+
+            // Load the initial value.
+            asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);
+            asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);
+
+            // again:
+            sink.bind_label(again_label, state.ctrl_plane_mut());
+
+            // cmpxchg16b (mem)
+            asm::inst::lock_cmpxchg16b_m::new(
+                PairedGpr::from(dst_old_low),
+                PairedGpr::from(dst_old_high),
+                operand_low,
+                operand_high,
+                mem,
+            )
+            .emit(sink, info, state);
+
+            // jnz again
+            one_way_jmp(sink, CC::NZ, again_label);
+        }
+
+        Inst::ElfTlsGetAddr { symbol, dst } => {
+            let dst = dst.to_reg().to_reg();
+            debug_assert_eq!(dst, regs::rax());
+
+            // N.B.: Must be exactly this byte sequence; the linker requires it,
+            // because it must know how to rewrite the bytes.
+
+            // data16 lea gv@tlsgd(%rip),%rdi
+            sink.put1(0x66); // data16
+            sink.put1(0b01001000); // REX.W
+            sink.put1(0x8d); // LEA
+            sink.put1(0x3d); // ModRM byte
+            emit_reloc(sink, Reloc::ElfX86_64TlsGd, symbol, -4);
+            sink.put4(0); // offset
+
+            // data16 data16 callq __tls_get_addr-4
+            sink.put1(0x66); // data16
+            sink.put1(0x66); // data16
+            sink.put1(0b01001000); // REX.W
+            sink.put1(0xe8); // CALL
+            emit_reloc(
+                sink,
+                Reloc::X86CallPLTRel4,
+                &ExternalName::LibCall(LibCall::ElfTlsGetAddr),
+                -4,
+            );
+            sink.put4(0); // offset
+        }
+
+        Inst::MachOTlsGetAddr { symbol, dst } => {
+            let dst = dst.to_reg().to_reg();
+            debug_assert_eq!(dst, regs::rax());
+
+            // movq gv@tlv(%rip), %rdi
+            sink.put1(0x48); // REX.w
+            sink.put1(0x8b); // MOV
+            sink.put1(0x3d); // ModRM byte
+            emit_reloc(sink, Reloc::MachOX86_64Tlv, symbol, -4);
+            sink.put4(0); // offset
+
+            asm::inst::callq_m::new(asm::Amode::ImmReg {
+                base: Gpr::RDI,
+                simm32: asm::AmodeOffsetPlusKnownOffset::ZERO,
+                trap: None,
+            })
+            .emit(sink, info, state);
+        }
+
+        Inst::CoffTlsGetAddr { symbol, dst, tmp } => {
+            let dst = dst.to_reg().to_reg();
+            debug_assert_eq!(dst, regs::rax());
+
+            // tmp is used below directly as %rcx
+            let tmp = tmp.to_reg().to_reg();
+            debug_assert_eq!(tmp, regs::rcx());
+
+            // See: https://gcc.godbolt.org/z/M8or9x6ss
+            // And: https://github.com/bjorn3/rustc_codegen_cranelift/issues/388#issuecomment-532930282
+
+            // Emit the following sequence
+            // movl	(%rip), %eax          ; IMAGE_REL_AMD64_REL32	_tls_index
+            // movq	%gs:88, %rcx
+            // movq	(%rcx,%rax,8), %rax
+            // leaq	(%rax), %rax          ; Reloc: IMAGE_REL_AMD64_SECREL	symbol
+
+            // Load TLS index for current thread
+            // movl	(%rip), %eax
+            sink.put1(0x8b); // mov
+            sink.put1(0x05);
+            emit_reloc(
+                sink,
+                Reloc::X86PCRel4,
+                &ExternalName::KnownSymbol(KnownSymbol::CoffTlsIndex),
+                -4,
+            );
+            sink.put4(0); // offset
+
+            // movq	%gs:88, %rcx
+            // Load the TLS Storage Array pointer
+            // The gs segment register refers to the base address of the TEB on x64.
+            // 0x58 is the offset in the TEB for the ThreadLocalStoragePointer member on x64:
+            sink.put_data(&[
+                0x65, 0x48, // REX.W
+                0x8b, // MOV
+                0x0c, 0x25, 0x58, // 0x58 - ThreadLocalStoragePointer offset
+                0x00, 0x00, 0x00,
+            ]);
+
+            // movq	(%rcx,%rax,8), %rax
+            // Load the actual TLS entry for this thread.
+            // Computes ThreadLocalStoragePointer + _tls_index*8
+            sink.put_data(&[0x48, 0x8b, 0x04, 0xc1]);
+
+            // leaq	(%rax), %rax
+            sink.put1(0x48);
+            sink.put1(0x8d);
+            sink.put1(0x80);
+            emit_reloc(sink, Reloc::X86SecRel, symbol, 0);
+            sink.put4(0); // offset
+        }
+
+        Inst::Unwind { inst } => {
+            sink.add_unwind(inst.clone());
+        }
+
+        Inst::DummyUse { .. } => {
+            // Nothing.
+        }
+
+        Inst::LabelAddress { dst, label } => {
+            // Emit an LEA with a LabelUse given this label.
+            asm::inst::leaq_rm::new(*dst, Amode::rip_relative(*label)).emit(sink, info, state);
+        }
+
+        Inst::External { inst } => {
+            let frame = state.frame_layout();
+            emit_maybe_shrink(
+                inst,
+                &mut external::AsmCodeSink {
+                    sink,
+
+                    // These values are transcribed from what is happening in
+                    // `SyntheticAmode::finalize`. This, plus the `Into` logic
+                    // converting a `SyntheticAmode` to its external counterpart, are
+                    // necessary to communicate Cranelift's internal offsets to the
+                    // assembler; due to when Cranelift determines these offsets, this
+                    // happens quite late (i.e., here during emission).
+                    incoming_arg_offset: i32::try_from(
+                        frame.tail_args_size + frame.setup_area_size,
+                    )
+                    .unwrap(),
+                    slot_offset: i32::try_from(frame.outgoing_args_size).unwrap(),
+                },
+            );
+        }
+    }
+
+    state.clear_post_insn();
+}
+
+/// Emit the common sequence used for both direct and indirect tail calls:
+///
+/// * Copy the new frame's stack arguments over the top of our current frame.
+///
+/// * Restore the old frame pointer.
+///
+/// * Initialize the tail callee's stack pointer (simultaneously deallocating
+///   the temporary stack space we allocated when creating the new frame's stack
+///   arguments).
+///
+/// * Move the return address into its stack slot.
+fn emit_return_call_common_sequence<T>(
+    sink: &mut MachBuffer<Inst>,
+    info: &EmitInfo,
+    state: &mut EmitState,
+    call_info: &ReturnCallInfo<T>,
+) {
+    assert!(
+        info.flags.preserve_frame_pointers(),
+        "frame pointers aren't fundamentally required for tail calls, \
+                 but the current implementation relies on them being present"
+    );
+
+    let tmp = call_info.tmp.to_writable_reg();
+
+    for inst in
+        X64ABIMachineSpec::gen_clobber_restore(CallConv::Tail, &info.flags, state.frame_layout())
+    {
+        inst.emit(sink, info, state);
+    }
+
+    for inst in X64ABIMachineSpec::gen_epilogue_frame_restore(
+        CallConv::Tail,
+        &info.flags,
+        &info.isa_flags,
+        state.frame_layout(),
+    ) {
+        inst.emit(sink, info, state);
+    }
+
+    let incoming_args_diff = state.frame_layout().tail_args_size - call_info.new_stack_arg_size;
+    if incoming_args_diff > 0 {
+        // Move the saved return address up by `incoming_args_diff`.
+        let addr = Amode::imm_reg(0, regs::rsp());
+        asm::inst::movq_rm::new(tmp, addr).emit(sink, info, state);
+        asm::inst::movq_mr::new(
+            Amode::imm_reg(i32::try_from(incoming_args_diff).unwrap(), regs::rsp()),
+            Gpr::unwrap_new(tmp.to_reg()),
+        )
+        .emit(sink, info, state);
+
+        // Increment the stack pointer to shrink the argument area for the new
+        // call.
+        let rsp = Writable::from_reg(regs::rsp());
+        let incoming_args_diff = i32::try_from(incoming_args_diff)
+            .expect("`incoming_args_diff` is too large to fit in a 32-bit signed immediate");
+        Inst::addq_mi(rsp, incoming_args_diff).emit(sink, info, state);
+    }
+}
+
+/// Conveniene trait to have an `emit` method on all `asm::inst::*` variants.
+trait ExternalEmit {
+    fn emit(self, sink: &mut MachBuffer<Inst>, info: &EmitInfo, state: &mut EmitState);
+}
+
+impl<I> ExternalEmit for I
+where
+    I: Into<asm::inst::Inst<CraneliftRegisters>>,
+{
+    fn emit(self, sink: &mut MachBuffer<Inst>, info: &EmitInfo, state: &mut EmitState) {
+        Inst::External { inst: self.into() }.emit(sink, info, state)
+    }
+}
+
+/// Attempt to "shrink" the provided `inst`.
+///
+/// This function will inspect `inst` and attempt to return a new instruction
+/// which is equivalent semantically but will encode to a smaller binary
+/// representation. This is only done for instructions which require register
+/// allocation to have already happened, for example shrinking immediates should
+/// be done during instruction selection not at this point.
+///
+/// An example of this optimization is the `AND` instruction. The Intel manual
+/// has a smaller encoding for `AND AL, imm8` than it does for `AND r/m8, imm8`.
+/// Here the instructions are matched against and if regalloc state indicates
+/// that a smaller variant is available then that's swapped to instead.
+fn emit_maybe_shrink(inst: &AsmInst, sink: &mut impl asm::CodeSink) {
+    use cranelift_assembler_x64::GprMem;
+    use cranelift_assembler_x64::inst::*;
+
+    type R = CraneliftRegisters;
+    const RAX: PairedGpr = PairedGpr {
+        read: Gpr::RAX,
+        write: Writable::from_reg(Gpr::RAX),
+    };
+    const RAX_RM: GprMem<PairedGpr, Gpr> = GprMem::Gpr(RAX);
+
+    match *inst {
+        // and
+        Inst::andb_mi(andb_mi { rm8: RAX_RM, imm8 }) => andb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::andw_mi(andw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => andw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::andl_mi(andl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => andl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::andq_mi_sxl(andq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => andq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // or
+        Inst::orb_mi(orb_mi { rm8: RAX_RM, imm8 }) => orb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::orw_mi(orw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => orw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::orl_mi(orl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => orl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::orq_mi_sxl(orq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => orq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // xor
+        Inst::xorb_mi(xorb_mi { rm8: RAX_RM, imm8 }) => xorb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::xorw_mi(xorw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => xorw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::xorl_mi(xorl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => xorl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::xorq_mi_sxl(xorq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => xorq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // add
+        Inst::addb_mi(addb_mi { rm8: RAX_RM, imm8 }) => addb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::addw_mi(addw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => addw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::addl_mi(addl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => addl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::addq_mi_sxl(addq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => addq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // adc
+        Inst::adcb_mi(adcb_mi { rm8: RAX_RM, imm8 }) => adcb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::adcw_mi(adcw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => adcw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::adcl_mi(adcl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => adcl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::adcq_mi_sxl(adcq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => adcq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // sub
+        Inst::subb_mi(subb_mi { rm8: RAX_RM, imm8 }) => subb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::subw_mi(subw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => subw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::subl_mi(subl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => subl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::subq_mi_sxl(subq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => subq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // sbb
+        Inst::sbbb_mi(sbbb_mi { rm8: RAX_RM, imm8 }) => sbbb_i::<R>::new(RAX, imm8).encode(sink),
+        Inst::sbbw_mi(sbbw_mi {
+            rm16: RAX_RM,
+            imm16,
+        }) => sbbw_i::<R>::new(RAX, imm16).encode(sink),
+        Inst::sbbl_mi(sbbl_mi {
+            rm32: RAX_RM,
+            imm32,
+        }) => sbbl_i::<R>::new(RAX, imm32).encode(sink),
+        Inst::sbbq_mi_sxl(sbbq_mi_sxl {
+            rm64: RAX_RM,
+            imm32,
+        }) => sbbq_i_sxl::<R>::new(RAX, imm32).encode(sink),
+
+        // cmp
+        Inst::cmpb_mi(cmpb_mi {
+            rm8: GprMem::Gpr(Gpr::RAX),
+            imm8,
+        }) => cmpb_i::<R>::new(Gpr::RAX, imm8).encode(sink),
+        Inst::cmpw_mi(cmpw_mi {
+            rm16: GprMem::Gpr(Gpr::RAX),
+            imm16,
+        }) => cmpw_i::<R>::new(Gpr::RAX, imm16).encode(sink),
+        Inst::cmpl_mi(cmpl_mi {
+            rm32: GprMem::Gpr(Gpr::RAX),
+            imm32,
+        }) => cmpl_i::<R>::new(Gpr::RAX, imm32).encode(sink),
+        Inst::cmpq_mi(cmpq_mi {
+            rm64: GprMem::Gpr(Gpr::RAX),
+            imm32,
+        }) => cmpq_i::<R>::new(Gpr::RAX, imm32).encode(sink),
+
+        // test
+        Inst::testb_mi(testb_mi {
+            rm8: GprMem::Gpr(Gpr::RAX),
+            imm8,
+        }) => testb_i::<R>::new(Gpr::RAX, imm8).encode(sink),
+        Inst::testw_mi(testw_mi {
+            rm16: GprMem::Gpr(Gpr::RAX),
+            imm16,
+        }) => testw_i::<R>::new(Gpr::RAX, imm16).encode(sink),
+        Inst::testl_mi(testl_mi {
+            rm32: GprMem::Gpr(Gpr::RAX),
+            imm32,
+        }) => testl_i::<R>::new(Gpr::RAX, imm32).encode(sink),
+        Inst::testq_mi(testq_mi {
+            rm64: GprMem::Gpr(Gpr::RAX),
+            imm32,
+        }) => testq_i::<R>::new(Gpr::RAX, imm32).encode(sink),
+
+        // lea
+        Inst::leal_rm(leal_rm { r32, m32 }) => emit_lea(
+            r32,
+            m32,
+            sink,
+            |dst, amode, s| leal_rm::<R>::new(dst, amode).encode(s),
+            |dst, simm32, s| addl_mi::<R>::new(dst, simm32.cast_unsigned()).encode(s),
+            |dst, reg, s| addl_rm::<R>::new(dst, reg).encode(s),
+        ),
+        Inst::leaq_rm(leaq_rm { r64, m64 }) => emit_lea(
+            r64,
+            m64,
+            sink,
+            |dst, amode, s| leaq_rm::<R>::new(dst, amode).encode(s),
+            |dst, simm32, s| addq_mi_sxl::<R>::new(dst, simm32).encode(s),
+            |dst, reg, s| addq_rm::<R>::new(dst, reg).encode(s),
+        ),
+
+        // All other instructions fall through to here and cannot be shrunk, so
+        // return `false` to emit them as usual.
+        _ => inst.encode(sink),
+    }
+}
+
+/// If `lea` can actually get encoded as an `add` then do that instead.
+/// Currently all candidate `iadd`s become an `lea` pseudo-instruction here but
+/// maximizing the use of `lea` is not necessarily optimal. The `lea`
+/// instruction goes through dedicated address units on cores which are finite
+/// and disjoint from the general ALU, so if everything uses `lea` then those
+/// units can get saturated while leaving the ALU idle.
+///
+/// To help make use of more parts of a CPU, this attempts to use `add` when
+/// it's semantically equivalent to `lea`, or otherwise when the `dst` register
+/// is the same as the `base` or `index` register.
+///
+/// FIXME: ideally regalloc is informed of this constraint. Register allocation
+/// of `lea` should "attempt" to put the `base` in the same register as `dst`
+/// but not at the expense of generating a `mov` instruction. Currently that's
+/// not possible but perhaps one day it may be worth it.
+fn emit_lea<S>(
+    dst: asm::Gpr<WritableGpr>,
+    addr: asm::Amode<Gpr>,
+    sink: &mut S,
+    lea: fn(WritableGpr, asm::Amode<Gpr>, &mut S),
+    add_mi: fn(PairedGpr, i32, &mut S),
+    add_rm: fn(PairedGpr, Gpr, &mut S),
+) where
+    S: asm::CodeSink,
+{
+    match addr {
+        // If `base == dst` then this is `add dst, $imm`, so encode that
+        // instead.
+        asm::Amode::ImmReg {
+            base,
+            simm32:
+                asm::AmodeOffsetPlusKnownOffset {
+                    simm32,
+                    offset: None,
+                },
+            trap: None,
+        } if dst.as_ref().to_reg() == base => add_mi(
+            PairedGpr {
+                read: base,
+                write: *dst.as_ref(),
+            },
+            simm32.value(),
+            sink,
+        ),
+
+        // If the offset is 0 and the shift is a scale of 1, then:
+        //
+        // * If `base == dst`, then this is `addq dst, index`
+        // * If `index == dst`, then this is `addq dst, base`
+        asm::Amode::ImmRegRegShift {
+            base,
+            index,
+            scale: asm::Scale::One,
+            simm32: asm::AmodeOffset::ZERO,
+            trap: None,
+        } => {
+            if dst.as_ref().to_reg() == base {
+                add_rm(
+                    PairedGpr {
+                        read: base,
+                        write: *dst.as_ref(),
+                    },
+                    *index.as_ref(),
+                    sink,
+                )
+            } else if dst.as_ref().to_reg() == *index.as_ref() {
+                add_rm(
+                    PairedGpr {
+                        read: *index.as_ref(),
+                        write: *dst.as_ref(),
+                    },
+                    base,
+                    sink,
+                )
+            } else {
+                lea(*dst.as_ref(), addr, sink)
+            }
+        }
+
+        _ => lea(*dst.as_ref(), addr, sink),
+    }
+}