wasmtime 28.0.0 → 29.0.0

data/ext/cargo-vendor/cranelift-codegen-0.115.0/src/isa/s390x/inst/emit.rs

@@ -1,3426 +0,0 @@
-//! S390x ISA: binary code emission.
-
-use crate::ir::{self, LibCall, MemFlags, TrapCode};
-use crate::isa::s390x::inst::*;
-use crate::isa::s390x::settings as s390x_settings;
-use cranelift_control::ControlPlane;
-
-/// Debug macro for testing that a regpair is valid: that the high register is even, and the low
-/// register is one higher than the high register.
-macro_rules! debug_assert_valid_regpair {
-    ($hi:expr, $lo:expr) => {
-        if cfg!(debug_assertions) {
-            match ($hi.to_real_reg(), $lo.to_real_reg()) {
-                (Some(hi), Some(lo)) => {
-                    assert!(
-                        hi.hw_enc() % 2 == 0,
-                        "High register is not even: {}",
-                        show_reg($hi)
-                    );
-                    assert_eq!(
-                        hi.hw_enc() + 1,
-                        lo.hw_enc(),
-                        "Low register is not valid: {}, {}",
-                        show_reg($hi),
-                        show_reg($lo)
-                    );
-                }
-
-                _ => {
-                    panic!(
-                        "Expected real registers for {} {}",
-                        show_reg($hi),
-                        show_reg($lo)
-                    );
-                }
-            }
-        }
-    };
-}
-
-/// Type(s) of memory instructions available for mem_finalize.
-pub struct MemInstType {
-    /// True if 12-bit unsigned displacement is supported.
-    pub have_d12: bool,
-    /// True if 20-bit signed displacement is supported.
-    pub have_d20: bool,
-    /// True if PC-relative addressing is supported (memory access).
-    pub have_pcrel: bool,
-    /// True if PC-relative addressing is supported (load address).
-    pub have_unaligned_pcrel: bool,
-    /// True if an index register is supported.
-    pub have_index: bool,
-}
-
-/// Memory addressing mode finalization: convert "special" modes (e.g.,
-/// generic arbitrary stack offset) into real addressing modes, possibly by
-/// emitting some helper instructions that come immediately before the use
-/// of this amode.
-pub fn mem_finalize(
-    mem: &MemArg,
-    state: &EmitState,
-    mi: MemInstType,
-) -> (SmallVec<[Inst; 4]>, MemArg) {
-    let mut insts = SmallVec::new();
-
-    // Resolve virtual addressing modes.
-    let mem = match mem {
-        &MemArg::RegOffset { off, .. }
-        | &MemArg::InitialSPOffset { off }
-        | &MemArg::NominalSPOffset { off }
-        | &MemArg::SlotOffset { off } => {
-            let base = match mem {
-                &MemArg::RegOffset { reg, .. } => reg,
-                &MemArg::InitialSPOffset { .. }
-                | &MemArg::NominalSPOffset { .. }
-                | &MemArg::SlotOffset { .. } => stack_reg(),
-                _ => unreachable!(),
-            };
-            let adj = match mem {
-                &MemArg::InitialSPOffset { .. } => i64::from(
-                    state.frame_layout().clobber_size
-                        + state.frame_layout().fixed_frame_storage_size
-                        + state.frame_layout().outgoing_args_size
-                        + state.nominal_sp_offset,
-                ),
-                &MemArg::SlotOffset { .. } => {
-                    i64::from(state.frame_layout().outgoing_args_size + state.nominal_sp_offset)
-                }
-                &MemArg::NominalSPOffset { .. } => i64::from(state.nominal_sp_offset),
-                _ => 0,
-            };
-            let off = off + adj;
-
-            if let Some(disp) = UImm12::maybe_from_u64(off as u64) {
-                MemArg::BXD12 {
-                    base,
-                    index: zero_reg(),
-                    disp,
-                    flags: mem.get_flags(),
-                }
-            } else if let Some(disp) = SImm20::maybe_from_i64(off) {
-                MemArg::BXD20 {
-                    base,
-                    index: zero_reg(),
-                    disp,
-                    flags: mem.get_flags(),
-                }
-            } else {
-                let tmp = writable_spilltmp_reg();
-                assert!(base != tmp.to_reg());
-                if let Ok(imm) = i16::try_from(off) {
-                    insts.push(Inst::Mov64SImm16 { rd: tmp, imm });
-                } else if let Ok(imm) = i32::try_from(off) {
-                    insts.push(Inst::Mov64SImm32 { rd: tmp, imm });
-                } else {
-                    // The offset must be smaller than the stack frame size,
-                    // which the ABI code limits to 128 MB.
-                    unreachable!();
-                }
-                MemArg::reg_plus_reg(base, tmp.to_reg(), mem.get_flags())
-            }
-        }
-        _ => mem.clone(),
-    };
-
-    // If this addressing mode cannot be handled by the instruction, use load-address.
-    let need_load_address = match &mem {
-        &MemArg::Label { .. } | &MemArg::Symbol { .. } if !mi.have_pcrel => true,
-        &MemArg::Symbol { flags, .. } if !mi.have_unaligned_pcrel && !flags.aligned() => true,
-        &MemArg::BXD20 { .. } if !mi.have_d20 => true,
-        &MemArg::BXD12 { index, .. } | &MemArg::BXD20 { index, .. } if !mi.have_index => {
-            index != zero_reg()
-        }
-        _ => false,
-    };
-    let mem = if need_load_address {
-        let flags = mem.get_flags();
-        let tmp = writable_spilltmp_reg();
-        insts.push(Inst::LoadAddr { rd: tmp, mem });
-        MemArg::reg(tmp.to_reg(), flags)
-    } else {
-        mem
-    };
-
-    // Convert 12-bit displacement to 20-bit if required.
-    let mem = match &mem {
-        &MemArg::BXD12 {
-            base,
-            index,
-            disp,
-            flags,
-        } if !mi.have_d12 => {
-            assert!(mi.have_d20);
-            MemArg::BXD20 {
-                base,
-                index,
-                disp: SImm20::from_uimm12(disp),
-                flags,
-            }
-        }
-        _ => mem,
-    };
-
-    (insts, mem)
-}
-
-pub fn mem_emit(
-    rd: Reg,
-    mem: &MemArg,
-    opcode_rx: Option<u16>,
-    opcode_rxy: Option<u16>,
-    opcode_ril: Option<u16>,
-    add_trap: bool,
-    sink: &mut MachBuffer<Inst>,
-    emit_info: &EmitInfo,
-    state: &mut EmitState,
-) {
-    let (mem_insts, mem) = mem_finalize(
-        mem,
-        state,
-        MemInstType {
-            have_d12: opcode_rx.is_some(),
-            have_d20: opcode_rxy.is_some(),
-            have_pcrel: opcode_ril.is_some(),
-            have_unaligned_pcrel: opcode_ril.is_some() && !add_trap,
-            have_index: true,
-        },
-    );
-    for inst in mem_insts.into_iter() {
-        inst.emit(sink, emit_info, state);
-    }
-
-    if add_trap {
-        if let Some(trap_code) = mem.get_flags().trap_code() {
-            sink.add_trap(trap_code);
-        }
-    }
-
-    match &mem {
-        &MemArg::BXD12 {
-            base, index, disp, ..
-        } => {
-            put(
-                sink,
-                &enc_rx(opcode_rx.unwrap(), rd, base, index, disp.bits()),
-            );
-        }
-        &MemArg::BXD20 {
-            base, index, disp, ..
-        } => {
-            put(
-                sink,
-                &enc_rxy(opcode_rxy.unwrap(), rd, base, index, disp.bits()),
-            );
-        }
-        &MemArg::Label { target } => {
-            sink.use_label_at_offset(sink.cur_offset(), target, LabelUse::BranchRIL);
-            put(sink, &enc_ril_b(opcode_ril.unwrap(), rd, 0));
-        }
-        &MemArg::Symbol {
-            ref name, offset, ..
-        } => {
-            sink.add_reloc_at_offset(2, Reloc::S390xPCRel32Dbl, &**name, (offset + 2).into());
-            put(sink, &enc_ril_b(opcode_ril.unwrap(), rd, 0));
-        }
-        _ => unreachable!(),
-    }
-}
-
-pub fn mem_rs_emit(
-    rd: Reg,
-    rn: Reg,
-    mem: &MemArg,
-    opcode_rs: Option<u16>,
-    opcode_rsy: Option<u16>,
-    add_trap: bool,
-    sink: &mut MachBuffer<Inst>,
-    emit_info: &EmitInfo,
-    state: &mut EmitState,
-) {
-    let (mem_insts, mem) = mem_finalize(
-        mem,
-        state,
-        MemInstType {
-            have_d12: opcode_rs.is_some(),
-            have_d20: opcode_rsy.is_some(),
-            have_pcrel: false,
-            have_unaligned_pcrel: false,
-            have_index: false,
-        },
-    );
-    for inst in mem_insts.into_iter() {
-        inst.emit(sink, emit_info, state);
-    }
-
-    if add_trap {
-        if let Some(trap_code) = mem.get_flags().trap_code() {
-            sink.add_trap(trap_code);
-        }
-    }
-
-    match &mem {
-        &MemArg::BXD12 {
-            base, index, disp, ..
-        } => {
-            assert!(index == zero_reg());
-            put(sink, &enc_rs(opcode_rs.unwrap(), rd, rn, base, disp.bits()));
-        }
-        &MemArg::BXD20 {
-            base, index, disp, ..
-        } => {
-            assert!(index == zero_reg());
-            put(
-                sink,
-                &enc_rsy(opcode_rsy.unwrap(), rd, rn, base, disp.bits()),
-            );
-        }
-        _ => unreachable!(),
-    }
-}
-
-pub fn mem_imm8_emit(
-    imm: u8,
-    mem: &MemArg,
-    opcode_si: u16,
-    opcode_siy: u16,
-    add_trap: bool,
-    sink: &mut MachBuffer<Inst>,
-    emit_info: &EmitInfo,
-    state: &mut EmitState,
-) {
-    let (mem_insts, mem) = mem_finalize(
-        mem,
-        state,
-        MemInstType {
-            have_d12: true,
-            have_d20: true,
-            have_pcrel: false,
-            have_unaligned_pcrel: false,
-            have_index: false,
-        },
-    );
-    for inst in mem_insts.into_iter() {
-        inst.emit(sink, emit_info, state);
-    }
-
-    if add_trap {
-        if let Some(trap_code) = mem.get_flags().trap_code() {
-            sink.add_trap(trap_code);
-        }
-    }
-
-    match &mem {
-        &MemArg::BXD12 {
-            base, index, disp, ..
-        } => {
-            assert!(index == zero_reg());
-            put(sink, &enc_si(opcode_si, base, disp.bits(), imm));
-        }
-        &MemArg::BXD20 {
-            base, index, disp, ..
-        } => {
-            assert!(index == zero_reg());
-            put(sink, &enc_siy(opcode_siy, base, disp.bits(), imm));
-        }
-        _ => unreachable!(),
-    }
-}
-
-pub fn mem_imm16_emit(
-    imm: i16,
-    mem: &MemArg,
-    opcode_sil: u16,
-    add_trap: bool,
-    sink: &mut MachBuffer<Inst>,
-    emit_info: &EmitInfo,
-    state: &mut EmitState,
-) {
-    let (mem_insts, mem) = mem_finalize(
-        mem,
-        state,
-        MemInstType {
-            have_d12: true,
-            have_d20: false,
-            have_pcrel: false,
-            have_unaligned_pcrel: false,
-            have_index: false,
-        },
-    );
-    for inst in mem_insts.into_iter() {
-        inst.emit(sink, emit_info, state);
-    }
-
-    if add_trap {
-        if let Some(trap_code) = mem.get_flags().trap_code() {
-            sink.add_trap(trap_code);
-        }
-    }
-
-    match &mem {
-        &MemArg::BXD12 {
-            base, index, disp, ..
-        } => {
-            assert!(index == zero_reg());
-            put(sink, &enc_sil(opcode_sil, base, disp.bits(), imm));
-        }
-        _ => unreachable!(),
-    }
-}
-
-pub fn mem_vrx_emit(
-    rd: Reg,
-    mem: &MemArg,
-    opcode: u16,
-    m3: u8,
-    add_trap: bool,
-    sink: &mut MachBuffer<Inst>,
-    emit_info: &EmitInfo,
-    state: &mut EmitState,
-) {
-    let (mem_insts, mem) = mem_finalize(
-        mem,
-        state,
-        MemInstType {
-            have_d12: true,
-            have_d20: false,
-            have_pcrel: false,
-            have_unaligned_pcrel: false,
-            have_index: true,
-        },
-    );
-    for inst in mem_insts.into_iter() {
-        inst.emit(sink, emit_info, state);
-    }
-
-    if add_trap {
-        if let Some(trap_code) = mem.get_flags().trap_code() {
-            sink.add_trap(trap_code);
-        }
-    }
-
-    match &mem {
-        &MemArg::BXD12 {
-            base, index, disp, ..
-        } => {
-            put(sink, &enc_vrx(opcode, rd, base, index, disp.bits(), m3));
-        }
-        _ => unreachable!(),
-    }
-}
-
-//=============================================================================
-// Instructions and subcomponents: emission
-
-fn machreg_to_gpr(m: Reg) -> u8 {
-    assert_eq!(m.class(), RegClass::Int);
-    u8::try_from(m.to_real_reg().unwrap().hw_enc()).unwrap()
-}
-
-fn machreg_to_vr(m: Reg) -> u8 {
-    assert_eq!(m.class(), RegClass::Float);
-    u8::try_from(m.to_real_reg().unwrap().hw_enc()).unwrap()
-}
-
-fn machreg_to_fpr(m: Reg) -> u8 {
-    assert!(is_fpr(m));
-    u8::try_from(m.to_real_reg().unwrap().hw_enc()).unwrap()
-}
-
-fn machreg_to_gpr_or_fpr(m: Reg) -> u8 {
-    let reg = u8::try_from(m.to_real_reg().unwrap().hw_enc()).unwrap();
-    assert!(reg < 16);
-    reg
-}
-
-fn rxb(v1: Option<Reg>, v2: Option<Reg>, v3: Option<Reg>, v4: Option<Reg>) -> u8 {
-    let mut rxb = 0;
-
-    let is_high_vr = |reg| -> bool {
-        if let Some(reg) = reg {
-            if !is_fpr(reg) {
-                return true;
-            }
-        }
-        false
-    };
-
-    if is_high_vr(v1) {
-        rxb = rxb | 8;
-    }
-    if is_high_vr(v2) {
-        rxb = rxb | 4;
-    }
-    if is_high_vr(v3) {
-        rxb = rxb | 2;
-    }
-    if is_high_vr(v4) {
-        rxb = rxb | 1;
-    }
-
-    rxb
-}
-
-/// E-type instructions.
-///
-///   15
-///   opcode
-///        0
-///
-fn enc_e(opcode: u16) -> [u8; 2] {
-    let mut enc: [u8; 2] = [0; 2];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-
-    enc[0] = opcode1;
-    enc[1] = opcode2;
-    enc
-}
-
-/// RIa-type instructions.
-///
-///   31      23 19      15
-///   opcode1 r1 opcode2 i2
-///        24 20      16  0
-///
-fn enc_ri_a(opcode: u16, r1: Reg, i2: u16) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 4) & 0xff) as u8;
-    let opcode2 = (opcode & 0xf) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | opcode2;
-    enc[2..].copy_from_slice(&i2.to_be_bytes());
-    enc
-}
-
-/// RIb-type instructions.
-///
-///   31      23 19      15
-///   opcode1 r1 opcode2 ri2
-///        24 20      16   0
-///
-fn enc_ri_b(opcode: u16, r1: Reg, ri2: i32) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 4) & 0xff) as u8;
-    let opcode2 = (opcode & 0xf) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let ri2 = ((ri2 >> 1) & 0xffff) as u16;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | opcode2;
-    enc[2..].copy_from_slice(&ri2.to_be_bytes());
-    enc
-}
-
-/// RIc-type instructions.
-///
-///   31      23 19      15
-///   opcode1 m1 opcode2 ri2
-///        24 20      16   0
-///
-fn enc_ri_c(opcode: u16, m1: u8, ri2: i32) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 4) & 0xff) as u8;
-    let opcode2 = (opcode & 0xf) as u8;
-    let m1 = m1 & 0x0f;
-    let ri2 = ((ri2 >> 1) & 0xffff) as u16;
-
-    enc[0] = opcode1;
-    enc[1] = m1 << 4 | opcode2;
-    enc[2..].copy_from_slice(&ri2.to_be_bytes());
-    enc
-}
-
-/// RIEa-type instructions.
-///
-///   47      39 35 31 15 11 7
-///   opcode1 r1 -- i2 m3 -- opcode2
-///        40 36 32 16 12 8       0
-///
-fn enc_rie_a(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let m3 = m3 & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4;
-    enc[2..4].copy_from_slice(&i2.to_be_bytes());
-    enc[4] = m3 << 4;
-    enc[5] = opcode2;
-    enc
-}
-
-/// RIEd-type instructions.
-///
-///   47      39 35 31 15 7
-///   opcode1 r1 r3 i2 -- opcode2
-///        40 36 32 16  8       0
-///
-fn enc_rie_d(opcode: u16, r1: Reg, r3: Reg, i2: u16) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let r3 = machreg_to_gpr(r3) & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | r3;
-    enc[2..4].copy_from_slice(&i2.to_be_bytes());
-    enc[5] = opcode2;
-    enc
-}
-
-/// RIEf-type instructions.
-///
-///   47      39 35 31 23 15 7
-///   opcode1 r1 r2 i3 i4 i5 opcode2
-///        40 36 32 24 16  8       0
-///
-fn enc_rie_f(opcode: u16, r1: Reg, r2: Reg, i3: u8, i4: u8, i5: u8) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let r2 = machreg_to_gpr(r2) & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | r2;
-    enc[2] = i3;
-    enc[3] = i4;
-    enc[4] = i5;
-    enc[5] = opcode2;
-    enc
-}
-
-/// RIEg-type instructions.
-///
-///   47      39 35 31 15 7
-///   opcode1 r1 m3 i2 -- opcode2
-///        40 36 32 16  8       0
-///
-fn enc_rie_g(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let m3 = m3 & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | m3;
-    enc[2..4].copy_from_slice(&i2.to_be_bytes());
-    enc[5] = opcode2;
-    enc
-}
-
-/// RILa-type instructions.
-///
-///   47      39 35      31
-///   opcode1 r1 opcode2 i2
-///        40 36      32  0
-///
-fn enc_ril_a(opcode: u16, r1: Reg, i2: u32) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 4) & 0xff) as u8;
-    let opcode2 = (opcode & 0xf) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | opcode2;
-    enc[2..].copy_from_slice(&i2.to_be_bytes());
-    enc
-}
-
-/// RILb-type instructions.
-///
-///   47      39 35      31
-///   opcode1 r1 opcode2 ri2
-///        40 36      32   0
-///
-fn enc_ril_b(opcode: u16, r1: Reg, ri2: u32) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 4) & 0xff) as u8;
-    let opcode2 = (opcode & 0xf) as u8;
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let ri2 = ri2 >> 1;
-
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | opcode2;
-    enc[2..].copy_from_slice(&ri2.to_be_bytes());
-    enc
-}
-
-/// RILc-type instructions.
-///
-///   47      39 35      31
-///   opcode1 m1 opcode2 i2
-///        40 36      32  0
-///
-fn enc_ril_c(opcode: u16, m1: u8, ri2: u32) -> [u8; 6] {
-    let mut enc: [u8; 6] = [0; 6];
-    let opcode1 = ((opcode >> 4) & 0xff) as u8;
-    let opcode2 = (opcode & 0xf) as u8;
-    let m1 = m1 & 0x0f;
-    let ri2 = ri2 >> 1;
-
-    enc[0] = opcode1;
-    enc[1] = m1 << 4 | opcode2;
-    enc[2..].copy_from_slice(&ri2.to_be_bytes());
-    enc
-}
-
-/// RR-type instructions.
-///
-///   15     7  3
-///   opcode r1 r2
-///        8  4  0
-///
-fn enc_rr(opcode: u16, r1: Reg, r2: Reg) -> [u8; 2] {
-    let mut enc: [u8; 2] = [0; 2];
-    let opcode = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
-
-    enc[0] = opcode;
-    enc[1] = r1 << 4 | r2;
-    enc
-}
-
-/// RRD-type instructions.
-///
-///   31     15 11 7  3
-///   opcode r1 -- r3 r2
-///       16 12  8 4  0
-///
-fn enc_rrd(opcode: u16, r1: Reg, r2: Reg, r3: Reg) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_fpr(r1) & 0x0f;
-    let r2 = machreg_to_fpr(r2) & 0x0f;
-    let r3 = machreg_to_fpr(r3) & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = opcode2;
-    enc[2] = r1 << 4;
-    enc[3] = r3 << 4 | r2;
-    enc
-}
-
-/// RRE-type instructions.
-///
-///   31     15 7  3
-///   opcode -- r1 r2
-///       16  8  4  0
-///
-fn enc_rre(opcode: u16, r1: Reg, r2: Reg) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = opcode2;
-    enc[3] = r1 << 4 | r2;
-    enc
-}
-
-/// RRFa/b-type instructions.
-///
-///   31     15 11 7  3
-///   opcode r3 m4 r1 r2
-///       16 12  8  4  0
-///
-fn enc_rrf_ab(opcode: u16, r1: Reg, r2: Reg, r3: Reg, m4: u8) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
-    let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
-    let m4 = m4 & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = opcode2;
-    enc[2] = r3 << 4 | m4;
-    enc[3] = r1 << 4 | r2;
-    enc
-}
-
-/// RRFc/d/e-type instructions.
-///
-///   31     15 11 7  3
-///   opcode m3 m4 r1 r2
-///       16 12  8  4  0
-///
-fn enc_rrf_cde(opcode: u16, r1: Reg, r2: Reg, m3: u8, m4: u8) -> [u8; 4] {
-    let mut enc: [u8; 4] = [0; 4];
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
-    let m3 = m3 & 0x0f;
-    let m4 = m4 & 0x0f;
-
-    enc[0] = opcode1;
-    enc[1] = opcode2;
-    enc[2] = m3 << 4 | m4;
-    enc[3] = r1 << 4 | r2;
-    enc
-}
-
-/// RS-type instructions.
-///
-///   31     23 19 15 11
-///   opcode r1 r3 b2 d2
-///       24 20 16 12  0
-///
-fn enc_rs(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 4] {
-    let opcode = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let d2_lo = (d2 & 0xff) as u8;
-    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
-
-    let mut enc: [u8; 4] = [0; 4];
-    enc[0] = opcode;
-    enc[1] = r1 << 4 | r3;
-    enc[2] = b2 << 4 | d2_hi;
-    enc[3] = d2_lo;
-    enc
-}
-
-/// RSY-type instructions.
-///
-///   47      39 35 31 27  15  7
-///   opcode1 r1 r3 b2 dl2 dh2 opcode2
-///        40 36 32 28  16   8       0
-///
-fn enc_rsy(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let dl2_lo = (d2 & 0xff) as u8;
-    let dl2_hi = ((d2 >> 8) & 0x0f) as u8;
-    let dh2 = ((d2 >> 12) & 0xff) as u8;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | r3;
-    enc[2] = b2 << 4 | dl2_hi;
-    enc[3] = dl2_lo;
-    enc[4] = dh2;
-    enc[5] = opcode2;
-    enc
-}
-
-/// RX-type instructions.
-///
-///   31     23 19 15 11
-///   opcode r1 x2 b2 d2
-///       24 20 16 12  0
-///
-fn enc_rx(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 4] {
-    let opcode = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let x2 = machreg_to_gpr(x2) & 0x0f;
-    let d2_lo = (d2 & 0xff) as u8;
-    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
-
-    let mut enc: [u8; 4] = [0; 4];
-    enc[0] = opcode;
-    enc[1] = r1 << 4 | x2;
-    enc[2] = b2 << 4 | d2_hi;
-    enc[3] = d2_lo;
-    enc
-}
-
-/// RXY-type instructions.
-///
-///   47      39 35 31 27  15  7
-///   opcode1 r1 x2 b2 dl2 dh2 opcode2
-///        40 36 32 28  16   8       0
-///
-fn enc_rxy(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let x2 = machreg_to_gpr(x2) & 0x0f;
-    let dl2_lo = (d2 & 0xff) as u8;
-    let dl2_hi = ((d2 >> 8) & 0x0f) as u8;
-    let dh2 = ((d2 >> 12) & 0xff) as u8;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | x2;
-    enc[2] = b2 << 4 | dl2_hi;
-    enc[3] = dl2_lo;
-    enc[4] = dh2;
-    enc[5] = opcode2;
-    enc
-}
-
-/// SI-type instructions.
-///
-///   31     23 15 11
-///   opcode i2 b1 d1
-///       24 16 12  0
-///
-fn enc_si(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 4] {
-    let opcode = (opcode & 0xff) as u8;
-    let b1 = machreg_to_gpr(b1) & 0x0f;
-    let d1_lo = (d1 & 0xff) as u8;
-    let d1_hi = ((d1 >> 8) & 0x0f) as u8;
-
-    let mut enc: [u8; 4] = [0; 4];
-    enc[0] = opcode;
-    enc[1] = i2;
-    enc[2] = b1 << 4 | d1_hi;
-    enc[3] = d1_lo;
-    enc
-}
-
-/// SIL-type instructions.
-///
-///   47     31 27 15
-///   opcode b1 d1 i2
-///       32 28 16  0
-///
-fn enc_sil(opcode: u16, b1: Reg, d1: u32, i2: i16) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let b1 = machreg_to_gpr(b1) & 0x0f;
-    let d1_lo = (d1 & 0xff) as u8;
-    let d1_hi = ((d1 >> 8) & 0x0f) as u8;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = opcode2;
-    enc[2] = b1 << 4 | d1_hi;
-    enc[3] = d1_lo;
-    enc[4..].copy_from_slice(&i2.to_be_bytes());
-    enc
-}
-
-/// SIY-type instructions.
-///
-///   47      39 31 27  15  7
-///   opcode1 i2 b1 dl1 dh1 opcode2
-///        40 32 28  16   8       0
-///
-fn enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let b1 = machreg_to_gpr(b1) & 0x0f;
-    let dl1_lo = (d1 & 0xff) as u8;
-    let dl1_hi = ((d1 >> 8) & 0x0f) as u8;
-    let dh1 = ((d1 >> 12) & 0xff) as u8;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = i2;
-    enc[2] = b1 << 4 | dl1_hi;
-    enc[3] = dl1_lo;
-    enc[4] = dh1;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRIa-type instructions.
-///
-///   47      39 35 31 15 11  7
-///   opcode1 v1 -  i2 m3 rxb opcode2
-///        40 36 32 16 12   8       0
-///
-fn enc_vri_a(opcode: u16, v1: Reg, i2: u16, m3: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), None, None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let m3 = m3 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4;
-    enc[2..4].copy_from_slice(&i2.to_be_bytes());
-    enc[4] = m3 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRIb-type instructions.
-///
-///   47      39 35 31 23 15 11  7
-///   opcode1 v1 -  i2 i3 m4 rxb opcode2
-///        40 36 32 24 16 12   8       0
-///
-fn enc_vri_b(opcode: u16, v1: Reg, i2: u8, i3: u8, m4: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), None, None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let m4 = m4 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4;
-    enc[2] = i2;
-    enc[3] = i3;
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRIc-type instructions.
-///
-///   47      39 35 31 15 11  7
-///   opcode1 v1 v3 i2 m4 rxb opcode2
-///        40 36 32 16 12   8       0
-///
-fn enc_vri_c(opcode: u16, v1: Reg, i2: u16, v3: Reg, m4: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), Some(v3), None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let v3 = machreg_to_vr(v3) & 0x0f;
-    let m4 = m4 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | v3;
-    enc[2..4].copy_from_slice(&i2.to_be_bytes());
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRRa-type instructions.
-///
-///   47      39 35 31 23 19 15 11  7
-///   opcode1 v1 v2 -  m5 m3 m2 rxb opcode2
-///        40 36 32 24 20 16 12   8       0
-///
-fn enc_vrr_a(opcode: u16, v1: Reg, v2: Reg, m3: u8, m4: u8, m5: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), Some(v2), None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let v2 = machreg_to_vr(v2) & 0x0f;
-    let m3 = m3 & 0x0f;
-    let m4 = m4 & 0x0f;
-    let m5 = m5 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | v2;
-    enc[2] = 0;
-    enc[3] = m5 << 4 | m4;
-    enc[4] = m3 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRRb-type instructions.
-///
-///   47      39 35 31 27 23 19 15 11  7
-///   opcode1 v1 v2 v3 -  m5 -  m4 rxb opcode2
-///        40 36 32 28 24 20 16 12   8       0
-///
-fn enc_vrr_b(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), Some(v2), Some(v3), None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let v2 = machreg_to_vr(v2) & 0x0f;
-    let v3 = machreg_to_vr(v3) & 0x0f;
-    let m4 = m4 & 0x0f;
-    let m5 = m5 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | v2;
-    enc[2] = v3 << 4;
-    enc[3] = m5 << 4;
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRRc-type instructions.
-///
-///   47      39 35 31 27 23 19 15 11  7
-///   opcode1 v1 v2 v3 -  m6 m5 m4 rxb opcode2
-///        40 36 32 28 24 20 16 12   8       0
-///
-fn enc_vrr_c(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8, m6: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), Some(v2), Some(v3), None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let v2 = machreg_to_vr(v2) & 0x0f;
-    let v3 = machreg_to_vr(v3) & 0x0f;
-    let m4 = m4 & 0x0f;
-    let m5 = m5 & 0x0f;
-    let m6 = m6 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | v2;
-    enc[2] = v3 << 4;
-    enc[3] = m6 << 4 | m5;
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRRe-type instructions.
-///
-///   47      39 35 31 27 23 19 15 11  7
-///   opcode1 v1 v2 v3 m6 -  m5 v4 rxb opcode2
-///        40 36 32 28 24 20 16 12   8       0
-///
-fn enc_vrr_e(opcode: u16, v1: Reg, v2: Reg, v3: Reg, v4: Reg, m5: u8, m6: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), Some(v2), Some(v3), Some(v4));
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let v2 = machreg_to_vr(v2) & 0x0f;
-    let v3 = machreg_to_vr(v3) & 0x0f;
-    let v4 = machreg_to_vr(v4) & 0x0f;
-    let m5 = m5 & 0x0f;
-    let m6 = m6 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | v2;
-    enc[2] = v3 << 4 | m6;
-    enc[3] = m5;
-    enc[4] = v4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRRf-type instructions.
-///
-///   47      39 35 31 27 11  7
-///   opcode1 v1 r2 r3 -  rxb opcode2
-///        40 36 32 28 12   8       0
-///
-fn enc_vrr_f(opcode: u16, v1: Reg, r2: Reg, r3: Reg) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), None, None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let r2 = machreg_to_gpr(r2) & 0x0f;
-    let r3 = machreg_to_gpr(r3) & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | r2;
-    enc[2] = r3 << 4;
-    enc[4] = rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRSa-type instructions.
-///
-///   47      39 35 31 27 15 11  7
-///   opcode1 v1 v3 b2 d2 m4 rxb opcode2
-///        40 36 32 28 16 12   8       0
-///
-fn enc_vrs_a(opcode: u16, v1: Reg, b2: Reg, d2: u32, v3: Reg, m4: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), Some(v3), None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let v3 = machreg_to_vr(v3) & 0x0f;
-    let d2_lo = (d2 & 0xff) as u8;
-    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
-    let m4 = m4 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | v3;
-    enc[2] = b2 << 4 | d2_hi;
-    enc[3] = d2_lo;
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRSb-type instructions.
-///
-///   47      39 35 31 27 15 11  7
-///   opcode1 v1 r3 b2 d2 m4 rxb opcode2
-///        40 36 32 28 16 12   8       0
-///
-fn enc_vrs_b(opcode: u16, v1: Reg, b2: Reg, d2: u32, r3: Reg, m4: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), None, None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let r3 = machreg_to_gpr(r3) & 0x0f;
-    let d2_lo = (d2 & 0xff) as u8;
-    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
-    let m4 = m4 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | r3;
-    enc[2] = b2 << 4 | d2_hi;
-    enc[3] = d2_lo;
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRSc-type instructions.
-///
-///   47      39 35 31 27 15 11  7
-///   opcode1 r1 v3 b2 d2 m4 rxb opcode2
-///        40 36 32 28 16 12   8       0
-///
-fn enc_vrs_c(opcode: u16, r1: Reg, b2: Reg, d2: u32, v3: Reg, m4: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(None, Some(v3), None, None);
-    let r1 = machreg_to_gpr(r1) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let v3 = machreg_to_vr(v3) & 0x0f;
-    let d2_lo = (d2 & 0xff) as u8;
-    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
-    let m4 = m4 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = r1 << 4 | v3;
-    enc[2] = b2 << 4 | d2_hi;
-    enc[3] = d2_lo;
-    enc[4] = m4 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// VRX-type instructions.
-///
-///   47      39 35 31 27 15 11  7
-///   opcode1 v1 x2 b2 d2 m3 rxb opcode2
-///        40 36 32 28 16 12   8       0
-///
-fn enc_vrx(opcode: u16, v1: Reg, b2: Reg, x2: Reg, d2: u32, m3: u8) -> [u8; 6] {
-    let opcode1 = ((opcode >> 8) & 0xff) as u8;
-    let opcode2 = (opcode & 0xff) as u8;
-    let rxb = rxb(Some(v1), None, None, None);
-    let v1 = machreg_to_vr(v1) & 0x0f;
-    let b2 = machreg_to_gpr(b2) & 0x0f;
-    let x2 = machreg_to_gpr(x2) & 0x0f;
-    let d2_lo = (d2 & 0xff) as u8;
-    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
-    let m3 = m3 & 0x0f;
-
-    let mut enc: [u8; 6] = [0; 6];
-    enc[0] = opcode1;
-    enc[1] = v1 << 4 | x2;
-    enc[2] = b2 << 4 | d2_hi;
-    enc[3] = d2_lo;
-    enc[4] = m3 << 4 | rxb;
-    enc[5] = opcode2;
-    enc
-}
-
-/// Emit encoding to sink.
-fn put(sink: &mut MachBuffer<Inst>, enc: &[u8]) {
-    for byte in enc {
-        sink.put1(*byte);
-    }
-}
-
-/// Emit encoding to sink, adding a trap on the last byte.
-fn put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], trap_code: TrapCode) {
-    let len = enc.len();
-    for i in 0..len - 1 {
-        sink.put1(enc[i]);
-    }
-    sink.add_trap(trap_code);
-    sink.put1(enc[len - 1]);
-}
-
-/// State carried between emissions of a sequence of instructions.
-#[derive(Default, Clone, Debug)]
-pub struct EmitState {
-    /// Offset from the actual SP to the "nominal SP".  The latter is defined
-    /// as the value the stack pointer has after the prolog.  This offset is
-    /// normally always zero, except during a call sequence using the tail-call
-    /// ABI, between the AllocateArgs and the actual call instruction.
-    pub(crate) nominal_sp_offset: u32,
-
-    /// The user stack map for the upcoming instruction, as provided to
-    /// `pre_safepoint()`.
-    user_stack_map: Option<ir::UserStackMap>,
-
-    /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
-    /// optimized away at compiletime. See [cranelift_control].
-    ctrl_plane: ControlPlane,
-
-    frame_layout: FrameLayout,
-}
-
-impl MachInstEmitState<Inst> for EmitState {
-    fn new(abi: &Callee<S390xMachineDeps>, ctrl_plane: ControlPlane) -> Self {
-        EmitState {
-            nominal_sp_offset: 0,
-            user_stack_map: None,
-            ctrl_plane,
-            frame_layout: abi.frame_layout().clone(),
-        }
-    }
-
-    fn pre_safepoint(&mut self, user_stack_map: Option<ir::UserStackMap>) {
-        self.user_stack_map = user_stack_map;
-    }
-
-    fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
-        &mut self.ctrl_plane
-    }
-
-    fn take_ctrl_plane(self) -> ControlPlane {
-        self.ctrl_plane
-    }
-
-    fn frame_layout(&self) -> &FrameLayout {
-        &self.frame_layout
-    }
-}
-
-impl EmitState {
-    fn take_stack_map(&mut self) -> Option<ir::UserStackMap> {
-        self.user_stack_map.take()
-    }
-
-    fn clear_post_insn(&mut self) {
-        self.user_stack_map = None;
-    }
-}
-
-/// Constant state used during function compilation.
-pub struct EmitInfo {
-    isa_flags: s390x_settings::Flags,
-}
-
-impl EmitInfo {
-    pub(crate) fn new(isa_flags: s390x_settings::Flags) -> Self {
-        Self { isa_flags }
-    }
-}
-
-impl MachInstEmit for Inst {
-    type State = EmitState;
-    type Info = EmitInfo;
-
-    fn emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState) {
-        self.emit_with_alloc_consumer(sink, emit_info, state)
-    }
-
-    fn pretty_print_inst(&self, state: &mut EmitState) -> String {
-        self.print_with_state(state)
-    }
-}
-
-impl Inst {
-    fn emit_with_alloc_consumer(
-        &self,
-        sink: &mut MachBuffer<Inst>,
-        emit_info: &EmitInfo,
-        state: &mut EmitState,
-    ) {
-        // Verify that we can emit this Inst in the current ISA
-        let matches_isa_flags = |iset_requirement: &InstructionSet| -> bool {
-            match iset_requirement {
-                // Baseline ISA is z14
-                InstructionSet::Base => true,
-                // Miscellaneous-Instruction-Extensions Facility 2 (z15)
-                InstructionSet::MIE2 => emit_info.isa_flags.has_mie2(),
-                // Vector-Enhancements Facility 2 (z15)
-                InstructionSet::VXRS_EXT2 => emit_info.isa_flags.has_vxrs_ext2(),
-            }
-        };
-        let isa_requirements = self.available_in_isa();
-        if !matches_isa_flags(&isa_requirements) {
-            panic!(
-                "Cannot emit inst '{self:?}' for target; failed to match ISA requirements: {isa_requirements:?}"
-            )
-        }
-
-        // N.B.: we *must* not exceed the "worst-case size" used to compute
-        // where to insert islands, except when islands are explicitly triggered
-        // (with an `EmitIsland`). We check this in debug builds. This is `mut`
-        // to allow disabling the check for `JTSequence`, which is always
-        // emitted following an `EmitIsland`.
-        let mut start_off = sink.cur_offset();
-
-        match self {
-            &Inst::AluRRR { alu_op, rd, rn, rm } => {
-                let (opcode, have_rr) = match alu_op {
-                    ALUOp::Add32 => (0xb9f8, true),        // ARK
-                    ALUOp::Add64 => (0xb9e8, true),        // AGRK
-                    ALUOp::AddLogical32 => (0xb9fa, true), // ALRK
-                    ALUOp::AddLogical64 => (0xb9ea, true), // ALGRK
-                    ALUOp::Sub32 => (0xb9f9, true),        // SRK
-                    ALUOp::Sub64 => (0xb9e9, true),        // SGRK
-                    ALUOp::SubLogical32 => (0xb9fb, true), // SLRK
-                    ALUOp::SubLogical64 => (0xb9eb, true), // SLGRK
-                    ALUOp::Mul32 => (0xb9fd, true),        // MSRKC
-                    ALUOp::Mul64 => (0xb9ed, true),        // MSGRKC
-                    ALUOp::And32 => (0xb9f4, true),        // NRK
-                    ALUOp::And64 => (0xb9e4, true),        // NGRK
-                    ALUOp::Orr32 => (0xb9f6, true),        // ORK
-                    ALUOp::Orr64 => (0xb9e6, true),        // OGRK
-                    ALUOp::Xor32 => (0xb9f7, true),        // XRK
-                    ALUOp::Xor64 => (0xb9e7, true),        // XGRK
-                    ALUOp::NotAnd32 => (0xb974, false),    // NNRK
-                    ALUOp::NotAnd64 => (0xb964, false),    // NNGRK
-                    ALUOp::NotOrr32 => (0xb976, false),    // NORK
-                    ALUOp::NotOrr64 => (0xb966, false),    // NOGRK
-                    ALUOp::NotXor32 => (0xb977, false),    // NXRK
-                    ALUOp::NotXor64 => (0xb967, false),    // NXGRK
-                    ALUOp::AndNot32 => (0xb9f5, false),    // NCRK
-                    ALUOp::AndNot64 => (0xb9e5, false),    // NCGRK
-                    ALUOp::OrrNot32 => (0xb975, false),    // OCRK
-                    ALUOp::OrrNot64 => (0xb965, false),    // OCGRK
-                    _ => unreachable!(),
-                };
-                if have_rr && rd.to_reg() == rn {
-                    let inst = Inst::AluRR {
-                        alu_op,
-                        rd,
-                        ri: rn,
-                        rm,
-                    };
-                    inst.emit(sink, emit_info, state);
-                } else {
-                    put(sink, &enc_rrf_ab(opcode, rd.to_reg(), rn, rm, 0));
-                }
-            }
-            &Inst::AluRRSImm16 {
-                alu_op,
-                rd,
-                rn,
-                imm,
-            } => {
-                if rd.to_reg() == rn {
-                    let inst = Inst::AluRSImm16 {
-                        alu_op,
-                        rd,
-                        ri: rn,
-                        imm,
-                    };
-                    inst.emit(sink, emit_info, state);
-                } else {
-                    let opcode = match alu_op {
-                        ALUOp::Add32 => 0xecd8, // AHIK
-                        ALUOp::Add64 => 0xecd9, // AGHIK
-                        _ => unreachable!(),
-                    };
-                    put(sink, &enc_rie_d(opcode, rd.to_reg(), rn, imm as u16));
-                }
-            }
-            &Inst::AluRR { alu_op, rd, ri, rm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let (opcode, is_rre) = match alu_op {
-                    ALUOp::Add32 => (0x1a, false),              // AR
-                    ALUOp::Add64 => (0xb908, true),             // AGR
-                    ALUOp::Add64Ext32 => (0xb918, true),        // AGFR
-                    ALUOp::AddLogical32 => (0x1e, false),       // ALR
-                    ALUOp::AddLogical64 => (0xb90a, true),      // ALGR
-                    ALUOp::AddLogical64Ext32 => (0xb91a, true), // ALGFR
-                    ALUOp::Sub32 => (0x1b, false),              // SR
-                    ALUOp::Sub64 => (0xb909, true),             // SGR
-                    ALUOp::Sub64Ext32 => (0xb919, true),        // SGFR
-                    ALUOp::SubLogical32 => (0x1f, false),       // SLR
-                    ALUOp::SubLogical64 => (0xb90b, true),      // SLGR
-                    ALUOp::SubLogical64Ext32 => (0xb91b, true), // SLGFR
-                    ALUOp::Mul32 => (0xb252, true),             // MSR
-                    ALUOp::Mul64 => (0xb90c, true),             // MSGR
-                    ALUOp::Mul64Ext32 => (0xb91c, true),        // MSGFR
-                    ALUOp::And32 => (0x14, false),              // NR
-                    ALUOp::And64 => (0xb980, true),             // NGR
-                    ALUOp::Orr32 => (0x16, false),              // OR
-                    ALUOp::Orr64 => (0xb981, true),             // OGR
-                    ALUOp::Xor32 => (0x17, false),              // XR
-                    ALUOp::Xor64 => (0xb982, true),             // XGR
-                    _ => unreachable!(),
-                };
-                if is_rre {
-                    put(sink, &enc_rre(opcode, rd.to_reg(), rm));
-                } else {
-                    put(sink, &enc_rr(opcode, rd.to_reg(), rm));
-                }
-            }
-            &Inst::AluRX {
-                alu_op,
-                rd,
-                ri,
-                ref mem,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-                let mem = mem.clone();
-
-                let (opcode_rx, opcode_rxy) = match alu_op {
-                    ALUOp::Add32 => (Some(0x5a), Some(0xe35a)),        // A(Y)
-                    ALUOp::Add32Ext16 => (Some(0x4a), Some(0xe37a)),   // AH(Y)
-                    ALUOp::Add64 => (None, Some(0xe308)),              // AG
-                    ALUOp::Add64Ext16 => (None, Some(0xe338)),         // AGH
-                    ALUOp::Add64Ext32 => (None, Some(0xe318)),         // AGF
-                    ALUOp::AddLogical32 => (Some(0x5e), Some(0xe35e)), // AL(Y)
-                    ALUOp::AddLogical64 => (None, Some(0xe30a)),       // ALG
-                    ALUOp::AddLogical64Ext32 => (None, Some(0xe31a)),  // ALGF
-                    ALUOp::Sub32 => (Some(0x5b), Some(0xe35b)),        // S(Y)
-                    ALUOp::Sub32Ext16 => (Some(0x4b), Some(0xe37b)),   // SH(Y)
-                    ALUOp::Sub64 => (None, Some(0xe309)),              // SG
-                    ALUOp::Sub64Ext16 => (None, Some(0xe339)),         // SGH
-                    ALUOp::Sub64Ext32 => (None, Some(0xe319)),         // SGF
-                    ALUOp::SubLogical32 => (Some(0x5f), Some(0xe35f)), // SL(Y)
-                    ALUOp::SubLogical64 => (None, Some(0xe30b)),       // SLG
-                    ALUOp::SubLogical64Ext32 => (None, Some(0xe31b)),  // SLGF
-                    ALUOp::Mul32 => (Some(0x71), Some(0xe351)),        // MS(Y)
-                    ALUOp::Mul32Ext16 => (Some(0x4c), Some(0xe37c)),   // MH(Y)
-                    ALUOp::Mul64 => (None, Some(0xe30c)),              // MSG
-                    ALUOp::Mul64Ext16 => (None, Some(0xe33c)),         // MSH
-                    ALUOp::Mul64Ext32 => (None, Some(0xe31c)),         // MSGF
-                    ALUOp::And32 => (Some(0x54), Some(0xe354)),        // N(Y)
-                    ALUOp::And64 => (None, Some(0xe380)),              // NG
-                    ALUOp::Orr32 => (Some(0x56), Some(0xe356)),        // O(Y)
-                    ALUOp::Orr64 => (None, Some(0xe381)),              // OG
-                    ALUOp::Xor32 => (Some(0x57), Some(0xe357)),        // X(Y)
-                    ALUOp::Xor64 => (None, Some(0xe382)),              // XG
-                    _ => unreachable!(),
-                };
-                let rd = rd.to_reg();
-                mem_emit(
-                    rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
-                );
-            }
-            &Inst::AluRSImm16 {
-                alu_op,
-                rd,
-                ri,
-                imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match alu_op {
-                    ALUOp::Add32 => 0xa7a, // AHI
-                    ALUOp::Add64 => 0xa7b, // AGHI
-                    ALUOp::Mul32 => 0xa7c, // MHI
-                    ALUOp::Mul64 => 0xa7d, // MGHI
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
-            }
-            &Inst::AluRSImm32 {
-                alu_op,
-                rd,
-                ri,
-                imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match alu_op {
-                    ALUOp::Add32 => 0xc29, // AFI
-                    ALUOp::Add64 => 0xc28, // AGFI
-                    ALUOp::Mul32 => 0xc21, // MSFI
-                    ALUOp::Mul64 => 0xc20, // MSGFI
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm as u32));
-            }
-            &Inst::AluRUImm32 {
-                alu_op,
-                rd,
-                ri,
-                imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match alu_op {
-                    ALUOp::AddLogical32 => 0xc2b, // ALFI
-                    ALUOp::AddLogical64 => 0xc2a, // ALGFI
-                    ALUOp::SubLogical32 => 0xc25, // SLFI
-                    ALUOp::SubLogical64 => 0xc24, // SLGFI
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm));
-            }
-            &Inst::AluRUImm16Shifted {
-                alu_op,
-                rd,
-                ri,
-                imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match (alu_op, imm.shift) {
-                    (ALUOp::And32, 0) => 0xa57, // NILL
-                    (ALUOp::And32, 1) => 0xa56, // NILH
-                    (ALUOp::And64, 0) => 0xa57, // NILL
-                    (ALUOp::And64, 1) => 0xa56, // NILH
-                    (ALUOp::And64, 2) => 0xa55, // NIHL
-                    (ALUOp::And64, 3) => 0xa54, // NIHL
-                    (ALUOp::Orr32, 0) => 0xa5b, // OILL
-                    (ALUOp::Orr32, 1) => 0xa5a, // OILH
-                    (ALUOp::Orr64, 0) => 0xa5b, // OILL
-                    (ALUOp::Orr64, 1) => 0xa5a, // OILH
-                    (ALUOp::Orr64, 2) => 0xa59, // OIHL
-                    (ALUOp::Orr64, 3) => 0xa58, // OIHH
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
-            }
-            &Inst::AluRUImm32Shifted {
-                alu_op,
-                rd,
-                ri,
-                imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match (alu_op, imm.shift) {
-                    (ALUOp::And32, 0) => 0xc0b, // NILF
-                    (ALUOp::And64, 0) => 0xc0b, // NILF
-                    (ALUOp::And64, 1) => 0xc0a, // NIHF
-                    (ALUOp::Orr32, 0) => 0xc0d, // OILF
-                    (ALUOp::Orr64, 0) => 0xc0d, // OILF
-                    (ALUOp::Orr64, 1) => 0xc0c, // OILF
-                    (ALUOp::Xor32, 0) => 0xc07, // XILF
-                    (ALUOp::Xor64, 0) => 0xc07, // XILF
-                    (ALUOp::Xor64, 1) => 0xc06, // XILH
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
-            }
-
-            &Inst::SMulWide { rd, rn, rm } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-
-                let opcode = 0xb9ec; // MGRK
-                put(sink, &enc_rrf_ab(opcode, rd1.to_reg(), rn, rm, 0));
-            }
-            &Inst::UMulWide { rd, ri, rn } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-                debug_assert_eq!(rd2.to_reg(), ri);
-
-                let opcode = 0xb986; // MLGR
-                put(sink, &enc_rre(opcode, rd1.to_reg(), rn));
-            }
-            &Inst::SDivMod32 { rd, ri, rn } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-                debug_assert_eq!(rd2.to_reg(), ri);
-
-                let opcode = 0xb91d; // DSGFR
-                let trap_code = TrapCode::INTEGER_DIVISION_BY_ZERO;
-                put_with_trap(sink, &enc_rre(opcode, rd1.to_reg(), rn), trap_code);
-            }
-            &Inst::SDivMod64 { rd, ri, rn } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-                debug_assert_eq!(rd2.to_reg(), ri);
-
-                let opcode = 0xb90d; // DSGR
-                let trap_code = TrapCode::INTEGER_DIVISION_BY_ZERO;
-                put_with_trap(sink, &enc_rre(opcode, rd1.to_reg(), rn), trap_code);
-            }
-            &Inst::UDivMod32 { rd, ri, rn } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-                let ri1 = ri.hi;
-                let ri2 = ri.lo;
-                debug_assert_eq!(rd1.to_reg(), ri1);
-                debug_assert_eq!(rd2.to_reg(), ri2);
-
-                let opcode = 0xb997; // DLR
-                let trap_code = TrapCode::INTEGER_DIVISION_BY_ZERO;
-                put_with_trap(sink, &enc_rre(opcode, rd1.to_reg(), rn), trap_code);
-            }
-            &Inst::UDivMod64 { rd, ri, rn } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-                let ri1 = ri.hi;
-                let ri2 = ri.lo;
-                debug_assert_eq!(rd1.to_reg(), ri1);
-                debug_assert_eq!(rd2.to_reg(), ri2);
-
-                let opcode = 0xb987; // DLGR
-                let trap_code = TrapCode::INTEGER_DIVISION_BY_ZERO;
-                put_with_trap(sink, &enc_rre(opcode, rd1.to_reg(), rn), trap_code);
-            }
-            &Inst::Flogr { rd, rn } => {
-                let rd1 = rd.hi;
-                let rd2 = rd.lo;
-                debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
-
-                let opcode = 0xb983; // FLOGR
-                put(sink, &enc_rre(opcode, rd1.to_reg(), rn));
-            }
-
-            &Inst::ShiftRR {
-                shift_op,
-                rd,
-                rn,
-                shift_imm,
-                shift_reg,
-            } => {
-                let opcode = match shift_op {
-                    ShiftOp::RotL32 => 0xeb1d, // RLL
-                    ShiftOp::RotL64 => 0xeb1c, // RLLG
-                    ShiftOp::LShL32 => 0xebdf, // SLLK  (SLL ?)
-                    ShiftOp::LShL64 => 0xeb0d, // SLLG
-                    ShiftOp::LShR32 => 0xebde, // SRLK  (SRL ?)
-                    ShiftOp::LShR64 => 0xeb0c, // SRLG
-                    ShiftOp::AShR32 => 0xebdc, // SRAK  (SRA ?)
-                    ShiftOp::AShR64 => 0xeb0a, // SRAG
-                };
-                put(
-                    sink,
-                    &enc_rsy(opcode, rd.to_reg(), rn, shift_reg, shift_imm.into()),
-                );
-            }
-
-            &Inst::RxSBG {
-                op,
-                rd,
-                ri,
-                rn,
-                start_bit,
-                end_bit,
-                rotate_amt,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match op {
-                    RxSBGOp::Insert => 0xec59, // RISBGN
-                    RxSBGOp::And => 0xec54,    // RNSBG
-                    RxSBGOp::Or => 0xec56,     // ROSBG
-                    RxSBGOp::Xor => 0xec57,    // RXSBG
-                };
-                put(
-                    sink,
-                    &enc_rie_f(
-                        opcode,
-                        rd.to_reg(),
-                        rn,
-                        start_bit,
-                        end_bit,
-                        (rotate_amt as u8) & 63,
-                    ),
-                );
-            }
-
-            &Inst::RxSBGTest {
-                op,
-                rd,
-                rn,
-                start_bit,
-                end_bit,
-                rotate_amt,
-            } => {
-                let opcode = match op {
-                    RxSBGOp::And => 0xec54, // RNSBG
-                    RxSBGOp::Or => 0xec56,  // ROSBG
-                    RxSBGOp::Xor => 0xec57, // RXSBG
-                    _ => unreachable!(),
-                };
-                put(
-                    sink,
-                    &enc_rie_f(
-                        opcode,
-                        rd,
-                        rn,
-                        start_bit | 0x80,
-                        end_bit,
-                        (rotate_amt as u8) & 63,
-                    ),
-                );
-            }
-
-            &Inst::UnaryRR { op, rd, rn } => {
-                match op {
-                    UnaryOp::Abs32 => {
-                        let opcode = 0x10; // LPR
-                        put(sink, &enc_rr(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::Abs64 => {
-                        let opcode = 0xb900; // LPGR
-                        put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::Abs64Ext32 => {
-                        let opcode = 0xb910; // LPGFR
-                        put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::Neg32 => {
-                        let opcode = 0x13; // LCR
-                        put(sink, &enc_rr(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::Neg64 => {
-                        let opcode = 0xb903; // LCGR
-                        put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::Neg64Ext32 => {
-                        let opcode = 0xb913; // LCGFR
-                        put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::PopcntByte => {
-                        let opcode = 0xb9e1; // POPCNT
-                        put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 0, 0));
-                    }
-                    UnaryOp::PopcntReg => {
-                        let opcode = 0xb9e1; // POPCNT
-                        put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 8, 0));
-                    }
-                    UnaryOp::BSwap32 => {
-                        let opcode = 0xb91f; // LRVR
-                        put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-                    }
-                    UnaryOp::BSwap64 => {
-                        let opcode = 0xb90f; // LRVRG
-                        put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-                    }
-                }
-            }
-
-            &Inst::Extend {
-                rd,
-                rn,
-                signed,
-                from_bits,
-                to_bits,
-            } => {
-                let opcode = match (signed, from_bits, to_bits) {
-                    (_, 1, 32) => 0xb926,      // LBR
-                    (_, 1, 64) => 0xb906,      // LGBR
-                    (false, 8, 32) => 0xb994,  // LLCR
-                    (false, 8, 64) => 0xb984,  // LLGCR
-                    (true, 8, 32) => 0xb926,   // LBR
-                    (true, 8, 64) => 0xb906,   // LGBR
-                    (false, 16, 32) => 0xb995, // LLHR
-                    (false, 16, 64) => 0xb985, // LLGHR
-                    (true, 16, 32) => 0xb927,  // LHR
-                    (true, 16, 64) => 0xb907,  // LGHR
-                    (false, 32, 64) => 0xb916, // LLGFR
-                    (true, 32, 64) => 0xb914,  // LGFR
-                    _ => panic!(
-                        "Unsupported extend combination: signed = {signed}, from_bits = {from_bits}, to_bits = {to_bits}"
-                    ),
-                };
-                put(sink, &enc_rre(opcode, rd.to_reg(), rn));
-            }
-
-            &Inst::CmpRR { op, rn, rm } => {
-                let (opcode, is_rre) = match op {
-                    CmpOp::CmpS32 => (0x19, false),       // CR
-                    CmpOp::CmpS64 => (0xb920, true),      // CGR
-                    CmpOp::CmpS64Ext32 => (0xb930, true), // CGFR
-                    CmpOp::CmpL32 => (0x15, false),       // CLR
-                    CmpOp::CmpL64 => (0xb921, true),      // CLGR
-                    CmpOp::CmpL64Ext32 => (0xb931, true), // CLGFR
-                    _ => unreachable!(),
-                };
-                if is_rre {
-                    put(sink, &enc_rre(opcode, rn, rm));
-                } else {
-                    put(sink, &enc_rr(opcode, rn, rm));
-                }
-            }
-            &Inst::CmpRX { op, rn, ref mem } => {
-                let mem = mem.clone();
-
-                let (opcode_rx, opcode_rxy, opcode_ril) = match op {
-                    CmpOp::CmpS32 => (Some(0x59), Some(0xe359), Some(0xc6d)), // C(Y), CRL
-                    CmpOp::CmpS32Ext16 => (Some(0x49), Some(0xe379), Some(0xc65)), // CH(Y), CHRL
-                    CmpOp::CmpS64 => (None, Some(0xe320), Some(0xc68)),       // CG, CGRL
-                    CmpOp::CmpS64Ext16 => (None, Some(0xe334), Some(0xc64)),  // CGH, CGHRL
-                    CmpOp::CmpS64Ext32 => (None, Some(0xe330), Some(0xc6c)),  // CGF, CGFRL
-                    CmpOp::CmpL32 => (Some(0x55), Some(0xe355), Some(0xc6f)), // CL(Y), CLRL
-                    CmpOp::CmpL32Ext16 => (None, None, Some(0xc67)),          // CLHRL
-                    CmpOp::CmpL64 => (None, Some(0xe321), Some(0xc6a)),       // CLG, CLGRL
-                    CmpOp::CmpL64Ext16 => (None, None, Some(0xc66)),          // CLGHRL
-                    CmpOp::CmpL64Ext32 => (None, Some(0xe331), Some(0xc6e)),  // CLGF, CLGFRL
-                };
-                mem_emit(
-                    rn, &mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
-                );
-            }
-            &Inst::CmpRSImm16 { op, rn, imm } => {
-                let opcode = match op {
-                    CmpOp::CmpS32 => 0xa7e, // CHI
-                    CmpOp::CmpS64 => 0xa7f, // CGHI
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ri_a(opcode, rn, imm as u16));
-            }
-            &Inst::CmpRSImm32 { op, rn, imm } => {
-                let opcode = match op {
-                    CmpOp::CmpS32 => 0xc2d, // CFI
-                    CmpOp::CmpS64 => 0xc2c, // CGFI
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rn, imm as u32));
-            }
-            &Inst::CmpRUImm32 { op, rn, imm } => {
-                let opcode = match op {
-                    CmpOp::CmpL32 => 0xc2f, // CLFI
-                    CmpOp::CmpL64 => 0xc2e, // CLGFI
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rn, imm));
-            }
-            &Inst::CmpTrapRR {
-                op,
-                rn,
-                rm,
-                cond,
-                trap_code,
-            } => {
-                let opcode = match op {
-                    CmpOp::CmpS32 => 0xb972, // CRT
-                    CmpOp::CmpS64 => 0xb960, // CGRT
-                    CmpOp::CmpL32 => 0xb973, // CLRT
-                    CmpOp::CmpL64 => 0xb961, // CLGRT
-                    _ => unreachable!(),
-                };
-                put_with_trap(
-                    sink,
-                    &enc_rrf_cde(opcode, rn, rm, cond.bits(), 0),
-                    trap_code,
-                );
-            }
-            &Inst::CmpTrapRSImm16 {
-                op,
-                rn,
-                imm,
-                cond,
-                trap_code,
-            } => {
-                let opcode = match op {
-                    CmpOp::CmpS32 => 0xec72, // CIT
-                    CmpOp::CmpS64 => 0xec70, // CGIT
-                    _ => unreachable!(),
-                };
-                put_with_trap(
-                    sink,
-                    &enc_rie_a(opcode, rn, imm as u16, cond.bits()),
-                    trap_code,
-                );
-            }
-            &Inst::CmpTrapRUImm16 {
-                op,
-                rn,
-                imm,
-                cond,
-                trap_code,
-            } => {
-                let opcode = match op {
-                    CmpOp::CmpL32 => 0xec73, // CLFIT
-                    CmpOp::CmpL64 => 0xec71, // CLGIT
-                    _ => unreachable!(),
-                };
-                put_with_trap(sink, &enc_rie_a(opcode, rn, imm, cond.bits()), trap_code);
-            }
-
-            &Inst::AtomicRmw {
-                alu_op,
-                rd,
-                rn,
-                ref mem,
-            } => {
-                let mem = mem.clone();
-
-                let opcode = match alu_op {
-                    ALUOp::Add32 => 0xebf8,        // LAA
-                    ALUOp::Add64 => 0xebe8,        // LAAG
-                    ALUOp::AddLogical32 => 0xebfa, // LAAL
-                    ALUOp::AddLogical64 => 0xebea, // LAALG
-                    ALUOp::And32 => 0xebf4,        // LAN
-                    ALUOp::And64 => 0xebe4,        // LANG
-                    ALUOp::Orr32 => 0xebf6,        // LAO
-                    ALUOp::Orr64 => 0xebe6,        // LAOG
-                    ALUOp::Xor32 => 0xebf7,        // LAX
-                    ALUOp::Xor64 => 0xebe7,        // LAXG
-                    _ => unreachable!(),
-                };
-
-                let rd = rd.to_reg();
-                mem_rs_emit(
-                    rd,
-                    rn,
-                    &mem,
-                    None,
-                    Some(opcode),
-                    true,
-                    sink,
-                    emit_info,
-                    state,
-                );
-            }
-            &Inst::Loop { ref body, cond } => {
-                // This sequence is *one* instruction in the vcode, and is expanded only here at
-                // emission time, because it requires branching to internal labels.
-                let loop_label = sink.get_label();
-                let done_label = sink.get_label();
-
-                // Emit label at the start of the loop.
-                sink.bind_label(loop_label, &mut state.ctrl_plane);
-
-                for inst in (&body).into_iter() {
-                    match &inst {
-                        // Replace a CondBreak with a branch to done_label.
-                        &Inst::CondBreak { cond } => {
-                            let inst = Inst::OneWayCondBr {
-                                target: done_label,
-                                cond: *cond,
-                            };
-                            inst.emit_with_alloc_consumer(sink, emit_info, state);
-                        }
-                        _ => inst.emit_with_alloc_consumer(sink, emit_info, state),
-                    };
-                }
-
-                let inst = Inst::OneWayCondBr {
-                    target: loop_label,
-                    cond,
-                };
-                inst.emit(sink, emit_info, state);
-
-                // Emit label at the end of the loop.
-                sink.bind_label(done_label, &mut state.ctrl_plane);
-            }
-            &Inst::CondBreak { .. } => unreachable!(), // Only valid inside a Loop.
-            &Inst::AtomicCas32 {
-                rd,
-                ri,
-                rn,
-                ref mem,
-            }
-            | &Inst::AtomicCas64 {
-                rd,
-                ri,
-                rn,
-                ref mem,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-                let mem = mem.clone();
-
-                let (opcode_rs, opcode_rsy) = match self {
-                    &Inst::AtomicCas32 { .. } => (Some(0xba), Some(0xeb14)), // CS(Y)
-                    &Inst::AtomicCas64 { .. } => (None, Some(0xeb30)),       // CSG
-                    _ => unreachable!(),
-                };
-
-                let rd = rd.to_reg();
-                mem_rs_emit(
-                    rd, rn, &mem, opcode_rs, opcode_rsy, true, sink, emit_info, state,
-                );
-            }
-            &Inst::Fence => {
-                put(sink, &enc_e(0x07e0));
-            }
-
-            &Inst::Load32 { rd, ref mem }
-            | &Inst::Load32ZExt8 { rd, ref mem }
-            | &Inst::Load32SExt8 { rd, ref mem }
-            | &Inst::Load32ZExt16 { rd, ref mem }
-            | &Inst::Load32SExt16 { rd, ref mem }
-            | &Inst::Load64 { rd, ref mem }
-            | &Inst::Load64ZExt8 { rd, ref mem }
-            | &Inst::Load64SExt8 { rd, ref mem }
-            | &Inst::Load64ZExt16 { rd, ref mem }
-            | &Inst::Load64SExt16 { rd, ref mem }
-            | &Inst::Load64ZExt32 { rd, ref mem }
-            | &Inst::Load64SExt32 { rd, ref mem }
-            | &Inst::LoadRev16 { rd, ref mem }
-            | &Inst::LoadRev32 { rd, ref mem }
-            | &Inst::LoadRev64 { rd, ref mem } => {
-                let mem = mem.clone();
-
-                let (opcode_rx, opcode_rxy, opcode_ril) = match self {
-                    &Inst::Load32 { .. } => (Some(0x58), Some(0xe358), Some(0xc4d)), // L(Y), LRL
-                    &Inst::Load32ZExt8 { .. } => (None, Some(0xe394), None),         // LLC
-                    &Inst::Load32SExt8 { .. } => (None, Some(0xe376), None),         // LB
-                    &Inst::Load32ZExt16 { .. } => (None, Some(0xe395), Some(0xc42)), // LLH, LLHRL
-                    &Inst::Load32SExt16 { .. } => (Some(0x48), Some(0xe378), Some(0xc45)), // LH(Y), LHRL
-                    &Inst::Load64 { .. } => (None, Some(0xe304), Some(0xc48)), // LG, LGRL
-                    &Inst::Load64ZExt8 { .. } => (None, Some(0xe390), None),   // LLGC
-                    &Inst::Load64SExt8 { .. } => (None, Some(0xe377), None),   // LGB
-                    &Inst::Load64ZExt16 { .. } => (None, Some(0xe391), Some(0xc46)), // LLGH, LLGHRL
-                    &Inst::Load64SExt16 { .. } => (None, Some(0xe315), Some(0xc44)), // LGH, LGHRL
-                    &Inst::Load64ZExt32 { .. } => (None, Some(0xe316), Some(0xc4e)), // LLGF, LLGFRL
-                    &Inst::Load64SExt32 { .. } => (None, Some(0xe314), Some(0xc4c)), // LGF, LGFRL
-                    &Inst::LoadRev16 { .. } => (None, Some(0xe31f), None),     // LRVH
-                    &Inst::LoadRev32 { .. } => (None, Some(0xe31e), None),     // LRV
-                    &Inst::LoadRev64 { .. } => (None, Some(0xe30f), None),     // LRVG
-                    _ => unreachable!(),
-                };
-                let rd = rd.to_reg();
-                mem_emit(
-                    rd, &mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
-                );
-            }
-
-            &Inst::Store8 { rd, ref mem }
-            | &Inst::Store16 { rd, ref mem }
-            | &Inst::Store32 { rd, ref mem }
-            | &Inst::Store64 { rd, ref mem }
-            | &Inst::StoreRev16 { rd, ref mem }
-            | &Inst::StoreRev32 { rd, ref mem }
-            | &Inst::StoreRev64 { rd, ref mem } => {
-                let mem = mem.clone();
-
-                let (opcode_rx, opcode_rxy, opcode_ril) = match self {
-                    &Inst::Store8 { .. } => (Some(0x42), Some(0xe372), None), // STC(Y)
-                    &Inst::Store16 { .. } => (Some(0x40), Some(0xe370), Some(0xc47)), // STH(Y), STHRL
-                    &Inst::Store32 { .. } => (Some(0x50), Some(0xe350), Some(0xc4f)), // ST(Y), STRL
-                    &Inst::Store64 { .. } => (None, Some(0xe324), Some(0xc4b)),       // STG, STGRL
-                    &Inst::StoreRev16 { .. } => (None, Some(0xe33f), None),           // STRVH
-                    &Inst::StoreRev32 { .. } => (None, Some(0xe33e), None),           // STRV
-                    &Inst::StoreRev64 { .. } => (None, Some(0xe32f), None),           // STRVG
-                    _ => unreachable!(),
-                };
-                mem_emit(
-                    rd, &mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
-                );
-            }
-            &Inst::StoreImm8 { imm, ref mem } => {
-                let mem = mem.clone();
-
-                let opcode_si = 0x92; // MVI
-                let opcode_siy = 0xeb52; // MVIY
-                mem_imm8_emit(
-                    imm, &mem, opcode_si, opcode_siy, true, sink, emit_info, state,
-                );
-            }
-            &Inst::StoreImm16 { imm, ref mem }
-            | &Inst::StoreImm32SExt16 { imm, ref mem }
-            | &Inst::StoreImm64SExt16 { imm, ref mem } => {
-                let mem = mem.clone();
-
-                let opcode = match self {
-                    &Inst::StoreImm16 { .. } => 0xe544,       // MVHHI
-                    &Inst::StoreImm32SExt16 { .. } => 0xe54c, // MVHI
-                    &Inst::StoreImm64SExt16 { .. } => 0xe548, // MVGHI
-                    _ => unreachable!(),
-                };
-                mem_imm16_emit(imm, &mem, opcode, true, sink, emit_info, state);
-            }
-
-            &Inst::LoadMultiple64 { rt, rt2, ref mem } => {
-                let mem = mem.clone();
-
-                let opcode = 0xeb04; // LMG
-                let rt = rt.to_reg();
-                let rt2 = rt2.to_reg();
-                mem_rs_emit(
-                    rt,
-                    rt2,
-                    &mem,
-                    None,
-                    Some(opcode),
-                    true,
-                    sink,
-                    emit_info,
-                    state,
-                );
-            }
-            &Inst::StoreMultiple64 { rt, rt2, ref mem } => {
-                let mem = mem.clone();
-
-                let opcode = 0xeb24; // STMG
-                mem_rs_emit(
-                    rt,
-                    rt2,
-                    &mem,
-                    None,
-                    Some(opcode),
-                    true,
-                    sink,
-                    emit_info,
-                    state,
-                );
-            }
-
-            &Inst::LoadAddr { rd, ref mem } => {
-                let mem = mem.clone();
-
-                let opcode_rx = Some(0x41); // LA
-                let opcode_rxy = Some(0xe371); // LAY
-                let opcode_ril = Some(0xc00); // LARL
-                let rd = rd.to_reg();
-                mem_emit(
-                    rd, &mem, opcode_rx, opcode_rxy, opcode_ril, false, sink, emit_info, state,
-                );
-            }
-
-            &Inst::Mov64 { rd, rm } => {
-                let opcode = 0xb904; // LGR
-                put(sink, &enc_rre(opcode, rd.to_reg(), rm));
-            }
-            &Inst::MovPReg { rd, rm } => {
-                Inst::Mov64 { rd, rm: rm.into() }.emit(sink, emit_info, state);
-            }
-            &Inst::Mov32 { rd, rm } => {
-                let opcode = 0x18; // LR
-                put(sink, &enc_rr(opcode, rd.to_reg(), rm));
-            }
-            &Inst::Mov32Imm { rd, imm } => {
-                let opcode = 0xc09; // IILF
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm));
-            }
-            &Inst::Mov32SImm16 { rd, imm } => {
-                let opcode = 0xa78; // LHI
-                put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
-            }
-            &Inst::Mov64SImm16 { rd, imm } => {
-                let opcode = 0xa79; // LGHI
-                put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
-            }
-            &Inst::Mov64SImm32 { rd, imm } => {
-                let opcode = 0xc01; // LGFI
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm as u32));
-            }
-            &Inst::CMov32 { rd, cond, ri, rm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = 0xb9f2; // LOCR
-                put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rm, cond.bits(), 0));
-            }
-            &Inst::CMov64 { rd, cond, ri, rm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = 0xb9e2; // LOCGR
-                put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rm, cond.bits(), 0));
-            }
-            &Inst::CMov32SImm16 { rd, cond, ri, imm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = 0xec42; // LOCHI
-                put(
-                    sink,
-                    &enc_rie_g(opcode, rd.to_reg(), imm as u16, cond.bits()),
-                );
-            }
-            &Inst::CMov64SImm16 { rd, cond, ri, imm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = 0xec46; // LOCGHI
-                put(
-                    sink,
-                    &enc_rie_g(opcode, rd.to_reg(), imm as u16, cond.bits()),
-                );
-            }
-            &Inst::Mov64UImm16Shifted { rd, imm } => {
-                let opcode = match imm.shift {
-                    0 => 0xa5f, // LLILL
-                    1 => 0xa5e, // LLILH
-                    2 => 0xa5d, // LLIHL
-                    3 => 0xa5c, // LLIHH
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
-            }
-            &Inst::Mov64UImm32Shifted { rd, imm } => {
-                let opcode = match imm.shift {
-                    0 => 0xc0f, // LLILF
-                    1 => 0xc0e, // LLIHF
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
-            }
-            &Inst::Insert64UImm16Shifted { rd, ri, imm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match imm.shift {
-                    0 => 0xa53, // IILL
-                    1 => 0xa52, // IILH
-                    2 => 0xa51, // IIHL
-                    3 => 0xa50, // IIHH
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
-            }
-            &Inst::Insert64UImm32Shifted { rd, ri, imm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match imm.shift {
-                    0 => 0xc09, // IILF
-                    1 => 0xc08, // IIHF
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
-            }
-            &Inst::LoadAR { rd, ar } => {
-                let opcode = 0xb24f; // EAR
-                put(sink, &enc_rre(opcode, rd.to_reg(), gpr(ar)));
-            }
-
-            &Inst::InsertAR { rd, ri, ar } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = 0xb24f; // EAR
-                put(sink, &enc_rre(opcode, rd.to_reg(), gpr(ar)));
-            }
-            &Inst::LoadSymbolReloc {
-                rd,
-                ref symbol_reloc,
-            } => {
-                let opcode = 0xa75; // BRAS
-                let reg = writable_spilltmp_reg().to_reg();
-                put(sink, &enc_ri_b(opcode, reg, 12));
-                let (reloc, name, offset) = match &**symbol_reloc {
-                    SymbolReloc::Absolute { name, offset } => (Reloc::Abs8, name, *offset),
-                    SymbolReloc::TlsGd { name } => (Reloc::S390xTlsGd64, name, 0),
-                };
-                sink.add_reloc(reloc, name, offset);
-                sink.put8(0);
-                let inst = Inst::Load64 {
-                    rd,
-                    mem: MemArg::reg(reg, MemFlags::trusted()),
-                };
-                inst.emit(sink, emit_info, state);
-            }
-
-            &Inst::FpuMove32 { rd, rn } => {
-                if is_fpr(rd.to_reg()) && is_fpr(rn) {
-                    let opcode = 0x38; // LER
-                    put(sink, &enc_rr(opcode, rd.to_reg(), rn));
-                } else {
-                    let opcode = 0xe756; // VLR
-                    put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, 0, 0, 0));
-                }
-            }
-            &Inst::FpuMove64 { rd, rn } => {
-                if is_fpr(rd.to_reg()) && is_fpr(rn) {
-                    let opcode = 0x28; // LDR
-                    put(sink, &enc_rr(opcode, rd.to_reg(), rn));
-                } else {
-                    let opcode = 0xe756; // VLR
-                    put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, 0, 0, 0));
-                }
-            }
-            &Inst::FpuCMov32 { rd, cond, ri, rm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                if is_fpr(rd.to_reg()) && is_fpr(rm) {
-                    let opcode = 0xa74; // BCR
-                    put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
-                    let opcode = 0x38; // LER
-                    put(sink, &enc_rr(opcode, rd.to_reg(), rm));
-                } else {
-                    let opcode = 0xa74; // BCR
-                    put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 6));
-                    let opcode = 0xe756; // VLR
-                    put(sink, &enc_vrr_a(opcode, rd.to_reg(), rm, 0, 0, 0));
-                }
-            }
-            &Inst::FpuCMov64 { rd, cond, ri, rm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                if is_fpr(rd.to_reg()) && is_fpr(rm) {
-                    let opcode = 0xa74; // BCR
-                    put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
-                    let opcode = 0x28; // LDR
-                    put(sink, &enc_rr(opcode, rd.to_reg(), rm));
-                } else {
-                    let opcode = 0xa74; // BCR
-                    put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 6));
-                    let opcode = 0xe756; // VLR
-                    put(sink, &enc_vrr_a(opcode, rd.to_reg(), rm, 0, 0, 0));
-                }
-            }
-            &Inst::LoadFpuConst32 { rd, const_data } => {
-                let opcode = 0xa75; // BRAS
-                let reg = writable_spilltmp_reg().to_reg();
-                put(sink, &enc_ri_b(opcode, reg, 8));
-                sink.put4(const_data.swap_bytes());
-                let inst = Inst::VecLoadLaneUndef {
-                    size: 32,
-                    rd,
-                    mem: MemArg::reg(reg, MemFlags::trusted()),
-                    lane_imm: 0,
-                };
-                inst.emit(sink, emit_info, state);
-            }
-            &Inst::LoadFpuConst64 { rd, const_data } => {
-                let opcode = 0xa75; // BRAS
-                let reg = writable_spilltmp_reg().to_reg();
-                put(sink, &enc_ri_b(opcode, reg, 12));
-                sink.put8(const_data.swap_bytes());
-                let inst = Inst::VecLoadLaneUndef {
-                    size: 64,
-                    rd,
-                    mem: MemArg::reg(reg, MemFlags::trusted()),
-                    lane_imm: 0,
-                };
-                inst.emit(sink, emit_info, state);
-            }
-            &Inst::FpuRR { fpu_op, rd, rn } => {
-                let (opcode, m3, m4, m5, opcode_fpr) = match fpu_op {
-                    FPUOp1::Abs32 => (0xe7cc, 2, 8, 2, Some(0xb300)), // WFPSO, LPEBR
-                    FPUOp1::Abs64 => (0xe7cc, 3, 8, 2, Some(0xb310)), // WFPSO, LPDBR
-                    FPUOp1::Abs32x4 => (0xe7cc, 2, 0, 2, None),       // VFPSO
-                    FPUOp1::Abs64x2 => (0xe7cc, 3, 0, 2, None),       // VFPSO
-                    FPUOp1::Neg32 => (0xe7cc, 2, 8, 0, Some(0xb303)), // WFPSO, LCEBR
-                    FPUOp1::Neg64 => (0xe7cc, 3, 8, 0, Some(0xb313)), // WFPSO, LCDBR
-                    FPUOp1::Neg32x4 => (0xe7cc, 2, 0, 0, None),       // VFPSO
-                    FPUOp1::Neg64x2 => (0xe7cc, 3, 0, 0, None),       // VFPSO
-                    FPUOp1::NegAbs32 => (0xe7cc, 2, 8, 1, Some(0xb301)), // WFPSO, LNEBR
-                    FPUOp1::NegAbs64 => (0xe7cc, 3, 8, 1, Some(0xb311)), // WFPSO, LNDBR
-                    FPUOp1::NegAbs32x4 => (0xe7cc, 2, 0, 1, None),    // VFPSO
-                    FPUOp1::NegAbs64x2 => (0xe7cc, 3, 0, 1, None),    // VFPSO
-                    FPUOp1::Sqrt32 => (0xe7ce, 2, 8, 0, Some(0xb314)), // WFSQ, SQEBR
-                    FPUOp1::Sqrt64 => (0xe7ce, 3, 8, 0, Some(0xb315)), // WFSQ, SQDBR
-                    FPUOp1::Sqrt32x4 => (0xe7ce, 2, 0, 0, None),      // VFSQ
-                    FPUOp1::Sqrt64x2 => (0xe7ce, 3, 0, 0, None),      // VFSQ
-                    FPUOp1::Cvt32To64 => (0xe7c4, 2, 8, 0, Some(0xb304)), // WFLL, LDEBR
-                    FPUOp1::Cvt32x4To64x2 => (0xe7c4, 2, 0, 0, None), // VFLL
-                };
-                if m4 == 8 && is_fpr(rd.to_reg()) && is_fpr(rn) {
-                    put(sink, &enc_rre(opcode_fpr.unwrap(), rd.to_reg(), rn));
-                } else {
-                    put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, m4, m5));
-                }
-            }
-            &Inst::FpuRRR { fpu_op, rd, rn, rm } => {
-                let (opcode, m4, m5, m6, opcode_fpr) = match fpu_op {
-                    FPUOp2::Add32 => (0xe7e3, 2, 8, 0, Some(0xb30a)), // WFA, AEBR
-                    FPUOp2::Add64 => (0xe7e3, 3, 8, 0, Some(0xb31a)), // WFA, ADBR
-                    FPUOp2::Add32x4 => (0xe7e3, 2, 0, 0, None),       // VFA
-                    FPUOp2::Add64x2 => (0xe7e3, 3, 0, 0, None),       // VFA
-                    FPUOp2::Sub32 => (0xe7e2, 2, 8, 0, Some(0xb30b)), // WFS, SEBR
-                    FPUOp2::Sub64 => (0xe7e2, 3, 8, 0, Some(0xb31b)), // WFS, SDBR
-                    FPUOp2::Sub32x4 => (0xe7e2, 2, 0, 0, None),       // VFS
-                    FPUOp2::Sub64x2 => (0xe7e2, 3, 0, 0, None),       // VFS
-                    FPUOp2::Mul32 => (0xe7e7, 2, 8, 0, Some(0xb317)), // WFM, MEEBR
-                    FPUOp2::Mul64 => (0xe7e7, 3, 8, 0, Some(0xb31c)), // WFM, MDBR
-                    FPUOp2::Mul32x4 => (0xe7e7, 2, 0, 0, None),       // VFM
-                    FPUOp2::Mul64x2 => (0xe7e7, 3, 0, 0, None),       // VFM
-                    FPUOp2::Div32 => (0xe7e5, 2, 8, 0, Some(0xb30d)), // WFD, DEBR
-                    FPUOp2::Div64 => (0xe7e5, 3, 8, 0, Some(0xb31d)), // WFD, DDBR
-                    FPUOp2::Div32x4 => (0xe7e5, 2, 0, 0, None),       // VFD
-                    FPUOp2::Div64x2 => (0xe7e5, 3, 0, 0, None),       // VFD
-                    FPUOp2::Max32 => (0xe7ef, 2, 8, 1, None),         // WFMAX
-                    FPUOp2::Max64 => (0xe7ef, 3, 8, 1, None),         // WFMAX
-                    FPUOp2::Max32x4 => (0xe7ef, 2, 0, 1, None),       // VFMAX
-                    FPUOp2::Max64x2 => (0xe7ef, 3, 0, 1, None),       // VFMAX
-                    FPUOp2::Min32 => (0xe7ee, 2, 8, 1, None),         // WFMIN
-                    FPUOp2::Min64 => (0xe7ee, 3, 8, 1, None),         // WFMIN
-                    FPUOp2::Min32x4 => (0xe7ee, 2, 0, 1, None),       // VFMIN
-                    FPUOp2::Min64x2 => (0xe7ee, 3, 0, 1, None),       // VFMIN
-                    FPUOp2::MaxPseudo32 => (0xe7ef, 2, 8, 3, None),   // WFMAX
-                    FPUOp2::MaxPseudo64 => (0xe7ef, 3, 8, 3, None),   // WFMAX
-                    FPUOp2::MaxPseudo32x4 => (0xe7ef, 2, 0, 3, None), // VFMAX
-                    FPUOp2::MaxPseudo64x2 => (0xe7ef, 3, 0, 3, None), // VFMAX
-                    FPUOp2::MinPseudo32 => (0xe7ee, 2, 8, 3, None),   // WFMIN
-                    FPUOp2::MinPseudo64 => (0xe7ee, 3, 8, 3, None),   // WFMIN
-                    FPUOp2::MinPseudo32x4 => (0xe7ee, 2, 0, 3, None), // VFMIN
-                    FPUOp2::MinPseudo64x2 => (0xe7ee, 3, 0, 3, None), // VFMIN
-                };
-                if m5 == 8 && opcode_fpr.is_some() && rd.to_reg() == rn && is_fpr(rn) && is_fpr(rm)
-                {
-                    put(sink, &enc_rre(opcode_fpr.unwrap(), rd.to_reg(), rm));
-                } else {
-                    put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, m5, m6));
-                }
-            }
-            &Inst::FpuRRRR {
-                fpu_op,
-                rd,
-                rn,
-                rm,
-                ra,
-            } => {
-                let (opcode, m5, m6, opcode_fpr) = match fpu_op {
-                    FPUOp3::MAdd32 => (0xe78f, 8, 2, Some(0xb30e)), // WFMA, MAEBR
-                    FPUOp3::MAdd64 => (0xe78f, 8, 3, Some(0xb31e)), // WFMA, MADBR
-                    FPUOp3::MAdd32x4 => (0xe78f, 0, 2, None),       // VFMA
-                    FPUOp3::MAdd64x2 => (0xe78f, 0, 3, None),       // VFMA
-                    FPUOp3::MSub32 => (0xe78e, 8, 2, Some(0xb30f)), // WFMS, MSEBR
-                    FPUOp3::MSub64 => (0xe78e, 8, 3, Some(0xb31f)), // WFMS, MSDBR
-                    FPUOp3::MSub32x4 => (0xe78e, 0, 2, None),       // VFMS
-                    FPUOp3::MSub64x2 => (0xe78e, 0, 3, None),       // VFMS
-                };
-                if m5 == 8 && rd.to_reg() == ra && is_fpr(rn) && is_fpr(rm) && is_fpr(ra) {
-                    put(sink, &enc_rrd(opcode_fpr.unwrap(), rd.to_reg(), rm, rn));
-                } else {
-                    put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, m5, m6));
-                }
-            }
-            &Inst::FpuRound { op, mode, rd, rn } => {
-                let mode = match mode {
-                    FpuRoundMode::Current => 0,
-                    FpuRoundMode::ToNearest => 1,
-                    FpuRoundMode::ShorterPrecision => 3,
-                    FpuRoundMode::ToNearestTiesToEven => 4,
-                    FpuRoundMode::ToZero => 5,
-                    FpuRoundMode::ToPosInfinity => 6,
-                    FpuRoundMode::ToNegInfinity => 7,
-                };
-                let (opcode, m3, m4, opcode_fpr) = match op {
-                    FpuRoundOp::Cvt64To32 => (0xe7c5, 3, 8, Some(0xb344)), // WFLR, LEDBR(A)
-                    FpuRoundOp::Cvt64x2To32x4 => (0xe7c5, 3, 0, None),     // VFLR
-                    FpuRoundOp::Round32 => (0xe7c7, 2, 8, Some(0xb357)),   // WFI, FIEBR
-                    FpuRoundOp::Round64 => (0xe7c7, 3, 8, Some(0xb35f)),   // WFI, FIDBR
-                    FpuRoundOp::Round32x4 => (0xe7c7, 2, 0, None),         // VFI
-                    FpuRoundOp::Round64x2 => (0xe7c7, 3, 0, None),         // VFI
-                    FpuRoundOp::ToSInt32 => (0xe7c2, 2, 8, None),          // WCSFP
-                    FpuRoundOp::ToSInt64 => (0xe7c2, 3, 8, None),          // WCSFP
-                    FpuRoundOp::ToUInt32 => (0xe7c0, 2, 8, None),          // WCLFP
-                    FpuRoundOp::ToUInt64 => (0xe7c0, 3, 8, None),          // WCLFP
-                    FpuRoundOp::ToSInt32x4 => (0xe7c2, 2, 0, None),        // VCSFP
-                    FpuRoundOp::ToSInt64x2 => (0xe7c2, 3, 0, None),        // VCSFP
-                    FpuRoundOp::ToUInt32x4 => (0xe7c0, 2, 0, None),        // VCLFP
-                    FpuRoundOp::ToUInt64x2 => (0xe7c0, 3, 0, None),        // VCLFP
-                    FpuRoundOp::FromSInt32 => (0xe7c3, 2, 8, None),        // WCFPS
-                    FpuRoundOp::FromSInt64 => (0xe7c3, 3, 8, None),        // WCFPS
-                    FpuRoundOp::FromUInt32 => (0xe7c1, 2, 8, None),        // WCFPL
-                    FpuRoundOp::FromUInt64 => (0xe7c1, 3, 8, None),        // WCFPL
-                    FpuRoundOp::FromSInt32x4 => (0xe7c3, 2, 0, None),      // VCFPS
-                    FpuRoundOp::FromSInt64x2 => (0xe7c3, 3, 0, None),      // VCFPS
-                    FpuRoundOp::FromUInt32x4 => (0xe7c1, 2, 0, None),      // VCFPL
-                    FpuRoundOp::FromUInt64x2 => (0xe7c1, 3, 0, None),      // VCFPL
-                };
-                if m4 == 8 && opcode_fpr.is_some() && is_fpr(rd.to_reg()) && is_fpr(rn) {
-                    put(
-                        sink,
-                        &enc_rrf_cde(opcode_fpr.unwrap(), rd.to_reg(), rn, mode, 0),
-                    );
-                } else {
-                    put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, m4, mode));
-                }
-            }
-            &Inst::FpuCmp32 { rn, rm } => {
-                if is_fpr(rn) && is_fpr(rm) {
-                    let opcode = 0xb309; // CEBR
-                    put(sink, &enc_rre(opcode, rn, rm));
-                } else {
-                    let opcode = 0xe7cb; // WFC
-                    put(sink, &enc_vrr_a(opcode, rn, rm, 2, 0, 0));
-                }
-            }
-            &Inst::FpuCmp64 { rn, rm } => {
-                if is_fpr(rn) && is_fpr(rm) {
-                    let opcode = 0xb319; // CDBR
-                    put(sink, &enc_rre(opcode, rn, rm));
-                } else {
-                    let opcode = 0xe7cb; // WFC
-                    put(sink, &enc_vrr_a(opcode, rn, rm, 3, 0, 0));
-                }
-            }
-
-            &Inst::VecRRR { op, rd, rn, rm } => {
-                let (opcode, m4) = match op {
-                    VecBinaryOp::Add8x16 => (0xe7f3, 0),       // VAB
-                    VecBinaryOp::Add16x8 => (0xe7f3, 1),       // VAH
-                    VecBinaryOp::Add32x4 => (0xe7f3, 2),       // VAF
-                    VecBinaryOp::Add64x2 => (0xe7f3, 3),       // VAG
-                    VecBinaryOp::Add128 => (0xe7f3, 4),        // VAQ
-                    VecBinaryOp::Sub8x16 => (0xe7f7, 0),       // VSB
-                    VecBinaryOp::Sub16x8 => (0xe7f7, 1),       // VSH
-                    VecBinaryOp::Sub32x4 => (0xe7f7, 2),       // VSF
-                    VecBinaryOp::Sub64x2 => (0xe7f7, 3),       // VSG
-                    VecBinaryOp::Sub128 => (0xe7f7, 4),        // VSQ
-                    VecBinaryOp::Mul8x16 => (0xe7a2, 0),       // VMLB
-                    VecBinaryOp::Mul16x8 => (0xe7a2, 1),       // VMLHW
-                    VecBinaryOp::Mul32x4 => (0xe7a2, 2),       // VMLF
-                    VecBinaryOp::UMulHi8x16 => (0xe7a1, 0),    // VMLHB
-                    VecBinaryOp::UMulHi16x8 => (0xe7a1, 1),    // VMLHH
-                    VecBinaryOp::UMulHi32x4 => (0xe7a1, 2),    // VMLHF
-                    VecBinaryOp::SMulHi8x16 => (0xe7a3, 0),    // VMHB
-                    VecBinaryOp::SMulHi16x8 => (0xe7a3, 1),    // VMHH
-                    VecBinaryOp::SMulHi32x4 => (0xe7a3, 2),    // VMHF
-                    VecBinaryOp::UMulEven8x16 => (0xe7a4, 0),  // VMLEB
-                    VecBinaryOp::UMulEven16x8 => (0xe7a4, 1),  // VMLEH
-                    VecBinaryOp::UMulEven32x4 => (0xe7a4, 2),  // VMLEF
-                    VecBinaryOp::SMulEven8x16 => (0xe7a6, 0),  // VMEB
-                    VecBinaryOp::SMulEven16x8 => (0xe7a6, 1),  // VMEH
-                    VecBinaryOp::SMulEven32x4 => (0xe7a6, 2),  // VMEF
-                    VecBinaryOp::UMulOdd8x16 => (0xe7a5, 0),   // VMLOB
-                    VecBinaryOp::UMulOdd16x8 => (0xe7a5, 1),   // VMLOH
-                    VecBinaryOp::UMulOdd32x4 => (0xe7a5, 2),   // VMLOF
-                    VecBinaryOp::SMulOdd8x16 => (0xe7a7, 0),   // VMOB
-                    VecBinaryOp::SMulOdd16x8 => (0xe7a7, 1),   // VMOH
-                    VecBinaryOp::SMulOdd32x4 => (0xe7a7, 2),   // VMOF
-                    VecBinaryOp::UMax8x16 => (0xe7fd, 0),      // VMXLB
-                    VecBinaryOp::UMax16x8 => (0xe7fd, 1),      // VMXLH
-                    VecBinaryOp::UMax32x4 => (0xe7fd, 2),      // VMXLF
-                    VecBinaryOp::UMax64x2 => (0xe7fd, 3),      // VMXLG
-                    VecBinaryOp::SMax8x16 => (0xe7ff, 0),      // VMXB
-                    VecBinaryOp::SMax16x8 => (0xe7ff, 1),      // VMXH
-                    VecBinaryOp::SMax32x4 => (0xe7ff, 2),      // VMXF
-                    VecBinaryOp::SMax64x2 => (0xe7ff, 3),      // VMXG
-                    VecBinaryOp::UMin8x16 => (0xe7fc, 0),      // VMNLB
-                    VecBinaryOp::UMin16x8 => (0xe7fc, 1),      // VMNLH
-                    VecBinaryOp::UMin32x4 => (0xe7fc, 2),      // VMNLF
-                    VecBinaryOp::UMin64x2 => (0xe7fc, 3),      // VMNLG
-                    VecBinaryOp::SMin8x16 => (0xe7fe, 0),      // VMNB
-                    VecBinaryOp::SMin16x8 => (0xe7fe, 1),      // VMNH
-                    VecBinaryOp::SMin32x4 => (0xe7fe, 2),      // VMNF
-                    VecBinaryOp::SMin64x2 => (0xe7fe, 3),      // VMNG
-                    VecBinaryOp::UAvg8x16 => (0xe7f0, 0),      // VAVGLB
-                    VecBinaryOp::UAvg16x8 => (0xe7f0, 1),      // VAVGLH
-                    VecBinaryOp::UAvg32x4 => (0xe7f0, 2),      // VAVGLF
-                    VecBinaryOp::UAvg64x2 => (0xe7f0, 3),      // VAVGLG
-                    VecBinaryOp::SAvg8x16 => (0xe7f2, 0),      // VAVGB
-                    VecBinaryOp::SAvg16x8 => (0xe7f2, 1),      // VAVGH
-                    VecBinaryOp::SAvg32x4 => (0xe7f2, 2),      // VAVGF
-                    VecBinaryOp::SAvg64x2 => (0xe7f2, 3),      // VAVGG
-                    VecBinaryOp::And128 => (0xe768, 0),        // VN
-                    VecBinaryOp::Orr128 => (0xe76a, 0),        // VO
-                    VecBinaryOp::Xor128 => (0xe76d, 0),        // VX
-                    VecBinaryOp::NotAnd128 => (0xe76e, 0),     // VNN
-                    VecBinaryOp::NotOrr128 => (0xe76b, 0),     // VNO
-                    VecBinaryOp::NotXor128 => (0xe76c, 0),     // VNX
-                    VecBinaryOp::AndNot128 => (0xe769, 0),     // VNC
-                    VecBinaryOp::OrrNot128 => (0xe76f, 0),     // VOC
-                    VecBinaryOp::BitPermute128 => (0xe785, 0), // VBPERM
-                    VecBinaryOp::LShLByByte128 => (0xe775, 0), // VSLB
-                    VecBinaryOp::LShRByByte128 => (0xe77d, 0), // VSRLB
-                    VecBinaryOp::AShRByByte128 => (0xe77f, 0), // VSRAB
-                    VecBinaryOp::LShLByBit128 => (0xe774, 0),  // VSL
-                    VecBinaryOp::LShRByBit128 => (0xe77c, 0),  // VSRL
-                    VecBinaryOp::AShRByBit128 => (0xe77e, 0),  // VSRA
-                    VecBinaryOp::Pack16x8 => (0xe794, 1),      // VPKH
-                    VecBinaryOp::Pack32x4 => (0xe794, 2),      // VPKF
-                    VecBinaryOp::Pack64x2 => (0xe794, 3),      // VPKG
-                    VecBinaryOp::PackUSat16x8 => (0xe795, 1),  // VPKLSH
-                    VecBinaryOp::PackUSat32x4 => (0xe795, 2),  // VPKLSF
-                    VecBinaryOp::PackUSat64x2 => (0xe795, 3),  // VPKLSG
-                    VecBinaryOp::PackSSat16x8 => (0xe797, 1),  // VPKSH
-                    VecBinaryOp::PackSSat32x4 => (0xe797, 2),  // VPKSF
-                    VecBinaryOp::PackSSat64x2 => (0xe797, 3),  // VPKSG
-                    VecBinaryOp::MergeLow8x16 => (0xe760, 0),  // VMRLB
-                    VecBinaryOp::MergeLow16x8 => (0xe760, 1),  // VMRLH
-                    VecBinaryOp::MergeLow32x4 => (0xe760, 2),  // VMRLF
-                    VecBinaryOp::MergeLow64x2 => (0xe760, 3),  // VMRLG
-                    VecBinaryOp::MergeHigh8x16 => (0xe761, 0), // VMRHB
-                    VecBinaryOp::MergeHigh16x8 => (0xe761, 1), // VMRHH
-                    VecBinaryOp::MergeHigh32x4 => (0xe761, 2), // VMRHF
-                    VecBinaryOp::MergeHigh64x2 => (0xe761, 3), // VMRHG
-                };
-
-                put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, 0));
-            }
-            &Inst::VecRR { op, rd, rn } => {
-                let (opcode, m3) = match op {
-                    VecUnaryOp::Abs8x16 => (0xe7df, 0),         // VLPB
-                    VecUnaryOp::Abs16x8 => (0xe7df, 1),         // VLPH
-                    VecUnaryOp::Abs32x4 => (0xe7df, 2),         // VLPF
-                    VecUnaryOp::Abs64x2 => (0xe7df, 3),         // VLPG
-                    VecUnaryOp::Neg8x16 => (0xe7de, 0),         // VLCB
-                    VecUnaryOp::Neg16x8 => (0xe7de, 1),         // VLCH
-                    VecUnaryOp::Neg32x4 => (0xe7de, 2),         // VLCF
-                    VecUnaryOp::Neg64x2 => (0xe7de, 3),         // VLCG
-                    VecUnaryOp::Popcnt8x16 => (0xe750, 0),      // VPOPCTB
-                    VecUnaryOp::Popcnt16x8 => (0xe750, 1),      // VPOPCTH
-                    VecUnaryOp::Popcnt32x4 => (0xe750, 2),      // VPOPCTF
-                    VecUnaryOp::Popcnt64x2 => (0xe750, 3),      // VPOPCTG
-                    VecUnaryOp::Clz8x16 => (0xe753, 0),         // VCLZB
-                    VecUnaryOp::Clz16x8 => (0xe753, 1),         // VCLZH
-                    VecUnaryOp::Clz32x4 => (0xe753, 2),         // VCLZF
-                    VecUnaryOp::Clz64x2 => (0xe753, 3),         // VCLZG
-                    VecUnaryOp::Ctz8x16 => (0xe752, 0),         // VCTZB
-                    VecUnaryOp::Ctz16x8 => (0xe752, 1),         // VCTZH
-                    VecUnaryOp::Ctz32x4 => (0xe752, 2),         // VCTZF
-                    VecUnaryOp::Ctz64x2 => (0xe752, 3),         // VCTZG
-                    VecUnaryOp::UnpackULow8x16 => (0xe7d4, 0),  // VUPLLB
-                    VecUnaryOp::UnpackULow16x8 => (0xe7d4, 1),  // VUPLLH
-                    VecUnaryOp::UnpackULow32x4 => (0xe7d4, 2),  // VUPLLF
-                    VecUnaryOp::UnpackUHigh8x16 => (0xe7d5, 0), // VUPLHB
-                    VecUnaryOp::UnpackUHigh16x8 => (0xe7d5, 1), // VUPLHH
-                    VecUnaryOp::UnpackUHigh32x4 => (0xe7d5, 2), // VUPLHF
-                    VecUnaryOp::UnpackSLow8x16 => (0xe7d6, 0),  // VUPLB
-                    VecUnaryOp::UnpackSLow16x8 => (0xe7d6, 1),  // VUPLH
-                    VecUnaryOp::UnpackSLow32x4 => (0xe7d6, 2),  // VUPLF
-                    VecUnaryOp::UnpackSHigh8x16 => (0xe7d7, 0), // VUPHB
-                    VecUnaryOp::UnpackSHigh16x8 => (0xe7d7, 1), // VUPHH
-                    VecUnaryOp::UnpackSHigh32x4 => (0xe7d7, 2), // VUPHF
-                };
-
-                put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, 0, 0));
-            }
-            &Inst::VecShiftRR {
-                shift_op,
-                rd,
-                rn,
-                shift_imm,
-                shift_reg,
-            } => {
-                let (opcode, m4) = match shift_op {
-                    VecShiftOp::RotL8x16 => (0xe733, 0), // VERLLB
-                    VecShiftOp::RotL16x8 => (0xe733, 1), // VERLLH
-                    VecShiftOp::RotL32x4 => (0xe733, 2), // VERLLF
-                    VecShiftOp::RotL64x2 => (0xe733, 3), // VERLLG
-                    VecShiftOp::LShL8x16 => (0xe730, 0), // VESLB
-                    VecShiftOp::LShL16x8 => (0xe730, 1), // VESLH
-                    VecShiftOp::LShL32x4 => (0xe730, 2), // VESLF
-                    VecShiftOp::LShL64x2 => (0xe730, 3), // VESLG
-                    VecShiftOp::LShR8x16 => (0xe738, 0), // VESRLB
-                    VecShiftOp::LShR16x8 => (0xe738, 1), // VESRLH
-                    VecShiftOp::LShR32x4 => (0xe738, 2), // VESRLF
-                    VecShiftOp::LShR64x2 => (0xe738, 3), // VESRLG
-                    VecShiftOp::AShR8x16 => (0xe73a, 0), // VESRAB
-                    VecShiftOp::AShR16x8 => (0xe73a, 1), // VESRAH
-                    VecShiftOp::AShR32x4 => (0xe73a, 2), // VESRAF
-                    VecShiftOp::AShR64x2 => (0xe73a, 3), // VESRAG
-                };
-                put(
-                    sink,
-                    &enc_vrs_a(opcode, rd.to_reg(), shift_reg, shift_imm.into(), rn, m4),
-                );
-            }
-            &Inst::VecSelect { rd, rn, rm, ra } => {
-                let opcode = 0xe78d; // VSEL
-                put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, 0, 0));
-            }
-            &Inst::VecPermute { rd, rn, rm, ra } => {
-                let opcode = 0xe78c; // VPERM
-                put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, 0, 0));
-            }
-            &Inst::VecPermuteDWImm {
-                rd,
-                rn,
-                rm,
-                idx1,
-                idx2,
-            } => {
-                let m4 = (idx1 & 1) * 4 + (idx2 & 1);
-
-                let opcode = 0xe784; // VPDI
-                put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, 0));
-            }
-            &Inst::VecIntCmp { op, rd, rn, rm } | &Inst::VecIntCmpS { op, rd, rn, rm } => {
-                let (opcode, m4) = match op {
-                    VecIntCmpOp::CmpEq8x16 => (0xe7f8, 0),  // VCEQB
-                    VecIntCmpOp::CmpEq16x8 => (0xe7f8, 1),  // VCEQH
-                    VecIntCmpOp::CmpEq32x4 => (0xe7f8, 2),  // VCEQF
-                    VecIntCmpOp::CmpEq64x2 => (0xe7f8, 3),  // VCEQG
-                    VecIntCmpOp::SCmpHi8x16 => (0xe7fb, 0), // VCHB
-                    VecIntCmpOp::SCmpHi16x8 => (0xe7fb, 1), // VCHH
-                    VecIntCmpOp::SCmpHi32x4 => (0xe7fb, 2), // VCHG
-                    VecIntCmpOp::SCmpHi64x2 => (0xe7fb, 3), // VCHG
-                    VecIntCmpOp::UCmpHi8x16 => (0xe7f9, 0), // VCHLB
-                    VecIntCmpOp::UCmpHi16x8 => (0xe7f9, 1), // VCHLH
-                    VecIntCmpOp::UCmpHi32x4 => (0xe7f9, 2), // VCHLG
-                    VecIntCmpOp::UCmpHi64x2 => (0xe7f9, 3), // VCHLG
-                };
-                let m5 = match self {
-                    &Inst::VecIntCmp { .. } => 0,
-                    &Inst::VecIntCmpS { .. } => 1,
-                    _ => unreachable!(),
-                };
-
-                put(sink, &enc_vrr_b(opcode, rd.to_reg(), rn, rm, m4, m5));
-            }
-            &Inst::VecFloatCmp { op, rd, rn, rm } | &Inst::VecFloatCmpS { op, rd, rn, rm } => {
-                let (opcode, m4) = match op {
-                    VecFloatCmpOp::CmpEq32x4 => (0xe7e8, 2),   // VFCESB
-                    VecFloatCmpOp::CmpEq64x2 => (0xe7e8, 3),   // VFCEDB
-                    VecFloatCmpOp::CmpHi32x4 => (0xe7eb, 2),   // VFCHSB
-                    VecFloatCmpOp::CmpHi64x2 => (0xe7eb, 3),   // VFCHDB
-                    VecFloatCmpOp::CmpHiEq32x4 => (0xe7ea, 2), // VFCHESB
-                    VecFloatCmpOp::CmpHiEq64x2 => (0xe7ea, 3), // VFCHEDB
-                };
-                let m6 = match self {
-                    &Inst::VecFloatCmp { .. } => 0,
-                    &Inst::VecFloatCmpS { .. } => 1,
-                    _ => unreachable!(),
-                };
-
-                put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, m6));
-            }
-            &Inst::VecInt128SCmpHi { tmp, rn, rm } | &Inst::VecInt128UCmpHi { tmp, rn, rm } => {
-                // Synthetic instruction to compare 128-bit values.
-                // Sets CC 1 if rn > rm, sets a different CC otherwise.
-
-                // Use VECTOR ELEMENT COMPARE to compare the high parts.
-                // Swap the inputs to get:
-                //    CC 1 if high(rn) > high(rm)
-                //    CC 2 if high(rn) < high(rm)
-                //    CC 0 if high(rn) == high(rm)
-                let (opcode, m3) = match self {
-                    &Inst::VecInt128SCmpHi { .. } => (0xe7db, 3), // VECG
-                    &Inst::VecInt128UCmpHi { .. } => (0xe7d9, 3), // VECLG
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_vrr_a(opcode, rm, rn, m3, 0, 0));
-
-                // If CC != 0, we'd done, so jump over the next instruction.
-                let opcode = 0xa74; // BCR
-                put(sink, &enc_ri_c(opcode, 7, 4 + 6));
-
-                // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
-                // Since we already know the high parts are equal, the CC
-                // result will only depend on the low parts:
-                //     CC 1 if low(rn) > low(rm)
-                //     CC 3 if low(rn) <= low(rm)
-                let inst = Inst::VecIntCmpS {
-                    op: VecIntCmpOp::UCmpHi64x2,
-                    // N.B.: This is the first write to tmp, and it happens
-                    // after all uses of rn and rm.  If this were to ever
-                    // change, tmp would have to become an early-def.
-                    rd: tmp,
-                    rn,
-                    rm,
-                };
-                inst.emit(sink, emit_info, state);
-            }
-
-            &Inst::VecLoad { rd, ref mem }
-            | &Inst::VecLoadRev { rd, ref mem }
-            | &Inst::VecLoadByte16Rev { rd, ref mem }
-            | &Inst::VecLoadByte32Rev { rd, ref mem }
-            | &Inst::VecLoadByte64Rev { rd, ref mem }
-            | &Inst::VecLoadElt16Rev { rd, ref mem }
-            | &Inst::VecLoadElt32Rev { rd, ref mem }
-            | &Inst::VecLoadElt64Rev { rd, ref mem } => {
-                let mem = mem.clone();
-
-                let (opcode, m3) = match self {
-                    &Inst::VecLoad { .. } => (0xe706, 0),          // VL
-                    &Inst::VecLoadRev { .. } => (0xe606, 4),       // VLBRQ
-                    &Inst::VecLoadByte16Rev { .. } => (0xe606, 1), // VLBRH
-                    &Inst::VecLoadByte32Rev { .. } => (0xe606, 2), // VLBRF
-                    &Inst::VecLoadByte64Rev { .. } => (0xe606, 3), // VLBRG
-                    &Inst::VecLoadElt16Rev { .. } => (0xe607, 1),  // VLERH
-                    &Inst::VecLoadElt32Rev { .. } => (0xe607, 2),  // VLERF
-                    &Inst::VecLoadElt64Rev { .. } => (0xe607, 3),  // VLERG
-                    _ => unreachable!(),
-                };
-                mem_vrx_emit(rd.to_reg(), &mem, opcode, m3, true, sink, emit_info, state);
-            }
-            &Inst::VecStore { rd, ref mem }
-            | &Inst::VecStoreRev { rd, ref mem }
-            | &Inst::VecStoreByte16Rev { rd, ref mem }
-            | &Inst::VecStoreByte32Rev { rd, ref mem }
-            | &Inst::VecStoreByte64Rev { rd, ref mem }
-            | &Inst::VecStoreElt16Rev { rd, ref mem }
-            | &Inst::VecStoreElt32Rev { rd, ref mem }
-            | &Inst::VecStoreElt64Rev { rd, ref mem } => {
-                let mem = mem.clone();
-
-                let (opcode, m3) = match self {
-                    &Inst::VecStore { .. } => (0xe70e, 0),          // VST
-                    &Inst::VecStoreRev { .. } => (0xe60e, 4),       // VSTBRQ
-                    &Inst::VecStoreByte16Rev { .. } => (0xe60e, 1), // VSTBRH
-                    &Inst::VecStoreByte32Rev { .. } => (0xe60e, 2), // VSTBRF
-                    &Inst::VecStoreByte64Rev { .. } => (0xe60e, 3), // VSTBRG
-                    &Inst::VecStoreElt16Rev { .. } => (0xe60f, 1),  // VSTERH
-                    &Inst::VecStoreElt32Rev { .. } => (0xe60f, 2),  // VSTERF
-                    &Inst::VecStoreElt64Rev { .. } => (0xe60f, 3),  // VSTERG
-                    _ => unreachable!(),
-                };
-                mem_vrx_emit(rd, &mem, opcode, m3, true, sink, emit_info, state);
-            }
-            &Inst::VecLoadReplicate { size, rd, ref mem }
-            | &Inst::VecLoadReplicateRev { size, rd, ref mem } => {
-                let mem = mem.clone();
-
-                let (opcode, m3) = match (self, size) {
-                    (&Inst::VecLoadReplicate { .. }, 8) => (0xe705, 0), // VLREPB
-                    (&Inst::VecLoadReplicate { .. }, 16) => (0xe705, 1), // VLREPH
-                    (&Inst::VecLoadReplicate { .. }, 32) => (0xe705, 2), // VLREPF
-                    (&Inst::VecLoadReplicate { .. }, 64) => (0xe705, 3), // VLREPG
-                    (&Inst::VecLoadReplicateRev { .. }, 16) => (0xe605, 1), // VLREPBRH
-                    (&Inst::VecLoadReplicateRev { .. }, 32) => (0xe605, 2), // VLREPBRF
-                    (&Inst::VecLoadReplicateRev { .. }, 64) => (0xe605, 3), // VLREPBRG
-                    _ => unreachable!(),
-                };
-                mem_vrx_emit(rd.to_reg(), &mem, opcode, m3, true, sink, emit_info, state);
-            }
-
-            &Inst::VecMov { rd, rn } => {
-                let opcode = 0xe756; // VLR
-                put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, 0, 0, 0));
-            }
-            &Inst::VecCMov { rd, cond, ri, rm } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = 0xa74; // BCR
-                put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 6));
-                let opcode = 0xe756; // VLR
-                put(sink, &enc_vrr_a(opcode, rd.to_reg(), rm, 0, 0, 0));
-            }
-            &Inst::MovToVec128 { rd, rn, rm } => {
-                let opcode = 0xe762; // VLVGP
-                put(sink, &enc_vrr_f(opcode, rd.to_reg(), rn, rm));
-            }
-            &Inst::VecLoadConst { rd, const_data } => {
-                let opcode = 0xa75; // BRAS
-                let reg = writable_spilltmp_reg().to_reg();
-                put(sink, &enc_ri_b(opcode, reg, 20));
-                for i in const_data.to_be_bytes().iter() {
-                    sink.put1(*i);
-                }
-                let inst = Inst::VecLoad {
-                    rd,
-                    mem: MemArg::reg(reg, MemFlags::trusted()),
-                };
-                inst.emit(sink, emit_info, state);
-            }
-            &Inst::VecLoadConstReplicate {
-                size,
-                rd,
-                const_data,
-            } => {
-                let opcode = 0xa75; // BRAS
-                let reg = writable_spilltmp_reg().to_reg();
-                put(sink, &enc_ri_b(opcode, reg, (4 + size / 8) as i32));
-                for i in 0..size / 8 {
-                    sink.put1((const_data >> (size - 8 - 8 * i)) as u8);
-                }
-                let inst = Inst::VecLoadReplicate {
-                    size,
-                    rd,
-                    mem: MemArg::reg(reg, MemFlags::trusted()),
-                };
-                inst.emit(sink, emit_info, state);
-            }
-            &Inst::VecImmByteMask { rd, mask } => {
-                let opcode = 0xe744; // VGBM
-                put(sink, &enc_vri_a(opcode, rd.to_reg(), mask, 0));
-            }
-            &Inst::VecImmBitMask {
-                size,
-                rd,
-                start_bit,
-                end_bit,
-            } => {
-                let (opcode, m4) = match size {
-                    8 => (0xe746, 0),  // VGMB
-                    16 => (0xe746, 1), // VGMH
-                    32 => (0xe746, 2), // VGMF
-                    64 => (0xe746, 3), // VGMG
-                    _ => unreachable!(),
-                };
-                put(
-                    sink,
-                    &enc_vri_b(opcode, rd.to_reg(), start_bit, end_bit, m4),
-                );
-            }
-            &Inst::VecImmReplicate { size, rd, imm } => {
-                let (opcode, m3) = match size {
-                    8 => (0xe745, 0),  // VREPIB
-                    16 => (0xe745, 1), // VREPIH
-                    32 => (0xe745, 2), // VREPIF
-                    64 => (0xe745, 3), // VREPIG
-                    _ => unreachable!(),
-                };
-                put(sink, &enc_vri_a(opcode, rd.to_reg(), imm as u16, m3));
-            }
-            &Inst::VecLoadLane {
-                size,
-                rd,
-                ri,
-                ref mem,
-                lane_imm,
-            }
-            | &Inst::VecLoadLaneRev {
-                size,
-                rd,
-                ri,
-                ref mem,
-                lane_imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-                let mem = mem.clone();
-
-                let opcode_vrx = match (self, size) {
-                    (&Inst::VecLoadLane { .. }, 8) => 0xe700,     // VLEB
-                    (&Inst::VecLoadLane { .. }, 16) => 0xe701,    // VLEH
-                    (&Inst::VecLoadLane { .. }, 32) => 0xe703,    // VLEF
-                    (&Inst::VecLoadLane { .. }, 64) => 0xe702,    // VLEG
-                    (&Inst::VecLoadLaneRev { .. }, 16) => 0xe601, // VLEBRH
-                    (&Inst::VecLoadLaneRev { .. }, 32) => 0xe603, // VLEBRF
-                    (&Inst::VecLoadLaneRev { .. }, 64) => 0xe602, // VLEBRG
-                    _ => unreachable!(),
-                };
-
-                let rd = rd.to_reg();
-                mem_vrx_emit(
-                    rd,
-                    &mem,
-                    opcode_vrx,
-                    lane_imm.into(),
-                    true,
-                    sink,
-                    emit_info,
-                    state,
-                );
-            }
-            &Inst::VecLoadLaneUndef {
-                size,
-                rd,
-                ref mem,
-                lane_imm,
-            }
-            | &Inst::VecLoadLaneRevUndef {
-                size,
-                rd,
-                ref mem,
-                lane_imm,
-            } => {
-                let mem = mem.clone();
-
-                let (opcode_vrx, opcode_rx, opcode_rxy) = match (self, size) {
-                    (&Inst::VecLoadLaneUndef { .. }, 8) => (0xe700, None, None), // VLEB
-                    (&Inst::VecLoadLaneUndef { .. }, 16) => (0xe701, None, None), // VLEH
-                    (&Inst::VecLoadLaneUndef { .. }, 32) => (0xe703, Some(0x78), Some(0xed64)), // VLEF, LE(Y)
-                    (&Inst::VecLoadLaneUndef { .. }, 64) => (0xe702, Some(0x68), Some(0xed65)), // VLEG, LD(Y)
-                    (&Inst::VecLoadLaneRevUndef { .. }, 16) => (0xe601, None, None), // VLEBRH
-                    (&Inst::VecLoadLaneRevUndef { .. }, 32) => (0xe603, None, None), // VLEBRF
-                    (&Inst::VecLoadLaneRevUndef { .. }, 64) => (0xe602, None, None), // VLEBRG
-                    _ => unreachable!(),
-                };
-
-                let rd = rd.to_reg();
-                if lane_imm == 0 && is_fpr(rd) && opcode_rx.is_some() {
-                    mem_emit(
-                        rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
-                    );
-                } else {
-                    mem_vrx_emit(
-                        rd,
-                        &mem,
-                        opcode_vrx,
-                        lane_imm.into(),
-                        true,
-                        sink,
-                        emit_info,
-                        state,
-                    );
-                }
-            }
-            &Inst::VecStoreLane {
-                size,
-                rd,
-                ref mem,
-                lane_imm,
-            }
-            | &Inst::VecStoreLaneRev {
-                size,
-                rd,
-                ref mem,
-                lane_imm,
-            } => {
-                let mem = mem.clone();
-
-                let (opcode_vrx, opcode_rx, opcode_rxy) = match (self, size) {
-                    (&Inst::VecStoreLane { .. }, 8) => (0xe708, None, None), // VSTEB
-                    (&Inst::VecStoreLane { .. }, 16) => (0xe709, None, None), // VSTEH
-                    (&Inst::VecStoreLane { .. }, 32) => (0xe70b, Some(0x70), Some(0xed66)), // VSTEF, STE(Y)
-                    (&Inst::VecStoreLane { .. }, 64) => (0xe70a, Some(0x60), Some(0xed67)), // VSTEG, STD(Y)
-                    (&Inst::VecStoreLaneRev { .. }, 16) => (0xe609, None, None), // VSTEBRH
-                    (&Inst::VecStoreLaneRev { .. }, 32) => (0xe60b, None, None), // VSTEBRF
-                    (&Inst::VecStoreLaneRev { .. }, 64) => (0xe60a, None, None), // VSTEBRG
-                    _ => unreachable!(),
-                };
-
-                if lane_imm == 0 && is_fpr(rd) && opcode_rx.is_some() {
-                    mem_emit(
-                        rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
-                    );
-                } else {
-                    mem_vrx_emit(
-                        rd,
-                        &mem,
-                        opcode_vrx,
-                        lane_imm.into(),
-                        true,
-                        sink,
-                        emit_info,
-                        state,
-                    );
-                }
-            }
-            &Inst::VecInsertLane {
-                size,
-                rd,
-                ri,
-                rn,
-                lane_imm,
-                lane_reg,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let (opcode_vrs, m4) = match size {
-                    8 => (0xe722, 0),  // VLVGB
-                    16 => (0xe722, 1), // VLVGH
-                    32 => (0xe722, 2), // VLVGF
-                    64 => (0xe722, 3), // VLVGG
-                    _ => unreachable!(),
-                };
-                put(
-                    sink,
-                    &enc_vrs_b(opcode_vrs, rd.to_reg(), lane_reg, lane_imm.into(), rn, m4),
-                );
-            }
-            &Inst::VecInsertLaneUndef {
-                size,
-                rd,
-                rn,
-                lane_imm,
-                lane_reg,
-            } => {
-                let (opcode_vrs, m4, opcode_rre) = match size {
-                    8 => (0xe722, 0, None),          // VLVGB
-                    16 => (0xe722, 1, None),         // VLVGH
-                    32 => (0xe722, 2, None),         // VLVGF
-                    64 => (0xe722, 3, Some(0xb3c1)), // VLVGG, LDGR
-                    _ => unreachable!(),
-                };
-                if opcode_rre.is_some()
-                    && lane_imm == 0
-                    && lane_reg == zero_reg()
-                    && is_fpr(rd.to_reg())
-                {
-                    put(sink, &enc_rre(opcode_rre.unwrap(), rd.to_reg(), rn));
-                } else {
-                    put(
-                        sink,
-                        &enc_vrs_b(opcode_vrs, rd.to_reg(), lane_reg, lane_imm.into(), rn, m4),
-                    );
-                }
-            }
-            &Inst::VecExtractLane {
-                size,
-                rd,
-                rn,
-                lane_imm,
-                lane_reg,
-            } => {
-                let (opcode_vrs, m4, opcode_rre) = match size {
-                    8 => (0xe721, 0, None),          // VLGVB
-                    16 => (0xe721, 1, None),         // VLGVH
-                    32 => (0xe721, 2, None),         // VLGVF
-                    64 => (0xe721, 3, Some(0xb3cd)), // VLGVG, LGDR
-                    _ => unreachable!(),
-                };
-                if opcode_rre.is_some() && lane_imm == 0 && lane_reg == zero_reg() && is_fpr(rn) {
-                    put(sink, &enc_rre(opcode_rre.unwrap(), rd.to_reg(), rn));
-                } else {
-                    put(
-                        sink,
-                        &enc_vrs_c(opcode_vrs, rd.to_reg(), lane_reg, lane_imm.into(), rn, m4),
-                    );
-                }
-            }
-            &Inst::VecInsertLaneImm {
-                size,
-                rd,
-                ri,
-                imm,
-                lane_imm,
-            } => {
-                debug_assert_eq!(rd.to_reg(), ri);
-
-                let opcode = match size {
-                    8 => 0xe740,  // VLEIB
-                    16 => 0xe741, // LEIVH
-                    32 => 0xe743, // VLEIF
-                    64 => 0xe742, // VLEIG
-                    _ => unreachable!(),
-                };
-                put(
-                    sink,
-                    &enc_vri_a(opcode, rd.to_reg(), imm as u16, lane_imm.into()),
-                );
-            }
-            &Inst::VecReplicateLane {
-                size,
-                rd,
-                rn,
-                lane_imm,
-            } => {
-                let (opcode, m4) = match size {
-                    8 => (0xe74d, 0),  // VREPB
-                    16 => (0xe74d, 1), // VREPH
-                    32 => (0xe74d, 2), // VREPF
-                    64 => (0xe74d, 3), // VREPG
-                    _ => unreachable!(),
-                };
-                put(
-                    sink,
-                    &enc_vri_c(opcode, rd.to_reg(), lane_imm.into(), rn, m4),
-                );
-            }
-
-            &Inst::AllocateArgs { size } => {
-                let inst = if let Ok(size) = i16::try_from(size) {
-                    Inst::AluRSImm16 {
-                        alu_op: ALUOp::Add64,
-                        rd: writable_stack_reg(),
-                        ri: stack_reg(),
-                        imm: -size,
-                    }
-                } else {
-                    Inst::AluRUImm32 {
-                        alu_op: ALUOp::SubLogical64,
-                        rd: writable_stack_reg(),
-                        ri: stack_reg(),
-                        imm: size,
-                    }
-                };
-                inst.emit(sink, emit_info, state);
-                state.nominal_sp_offset += size;
-            }
-            &Inst::Call { link, ref info } => {
-                let opcode = 0xc05; // BRASL
-
-                // Add relocation for target function.  This has to be done *before*
-                // the S390xTlsGdCall relocation if any, to ensure linker relaxation
-                // works correctly.
-                sink.add_reloc_at_offset(2, Reloc::S390xPLTRel32Dbl, &info.dest, 2);
-
-                if let Some(s) = state.take_stack_map() {
-                    let offset = sink.cur_offset() + 6;
-                    sink.push_user_stack_map(state, offset, s);
-                }
-
-                put(sink, &enc_ril_b(opcode, link.to_reg(), 0));
-                sink.add_call_site();
-
-                state.nominal_sp_offset -= info.callee_pop_size;
-            }
-            &Inst::CallInd { link, ref info } => {
-                if let Some(s) = state.take_stack_map() {
-                    let offset = sink.cur_offset() + 2;
-                    sink.push_user_stack_map(state, offset, s);
-                }
-
-                let opcode = 0x0d; // BASR
-                put(sink, &enc_rr(opcode, link.to_reg(), info.dest));
-                sink.add_call_site();
-
-                state.nominal_sp_offset -= info.callee_pop_size;
-            }
-            &Inst::ReturnCall { ref info } => {
-                for inst in S390xMachineDeps::gen_tail_epilogue(
-                    state.frame_layout(),
-                    info.callee_pop_size,
-                    None,
-                ) {
-                    inst.emit(sink, emit_info, state);
-                }
-
-                let opcode = 0xc04; // BCRL
-                sink.add_reloc_at_offset(2, Reloc::S390xPLTRel32Dbl, &info.dest, 2);
-                put(sink, &enc_ril_c(opcode, 15, 0));
-                sink.add_call_site();
-            }
-            &Inst::ReturnCallInd { ref info } => {
-                let mut rn = info.dest;
-                for inst in S390xMachineDeps::gen_tail_epilogue(
-                    state.frame_layout(),
-                    info.callee_pop_size,
-                    Some(&mut rn),
-                ) {
-                    inst.emit(sink, emit_info, state);
-                }
-
-                let opcode = 0x07; // BCR
-                put(sink, &enc_rr(opcode, gpr(15), rn));
-                sink.add_call_site();
-            }
-            &Inst::ElfTlsGetOffset { ref symbol, .. } => {
-                let opcode = 0xc05; // BRASL
-
-                // Add relocation for target function. This has to be done
-                // *before* the S390xTlsGdCall, to ensure linker relaxation
-                // works correctly.
-                let dest = ExternalName::LibCall(LibCall::ElfTlsGetOffset);
-                sink.add_reloc_at_offset(2, Reloc::S390xPLTRel32Dbl, &dest, 2);
-                match &**symbol {
-                    SymbolReloc::TlsGd { name } => sink.add_reloc(Reloc::S390xTlsGdCall, name, 0),
-                    _ => unreachable!(),
-                }
-
-                put(sink, &enc_ril_b(opcode, gpr(14), 0));
-                sink.add_call_site();
-            }
-            &Inst::Args { .. } => {}
-            &Inst::Rets { .. } => {}
-            &Inst::Ret { link } => {
-                let opcode = 0x07; // BCR
-                put(sink, &enc_rr(opcode, gpr(15), link));
-            }
-            &Inst::Jump { dest } => {
-                let off = sink.cur_offset();
-                // Indicate that the jump uses a label, if so, so that a fixup can occur later.
-                sink.use_label_at_offset(off, dest, LabelUse::BranchRIL);
-                sink.add_uncond_branch(off, off + 6, dest);
-                // Emit the jump itself.
-                let opcode = 0xc04; // BCRL
-                put(sink, &enc_ril_c(opcode, 15, 0));
-            }
-            &Inst::IndirectBr { rn, .. } => {
-                let opcode = 0x07; // BCR
-                put(sink, &enc_rr(opcode, gpr(15), rn));
-            }
-            &Inst::CondBr {
-                taken,
-                not_taken,
-                cond,
-            } => {
-                let opcode = 0xc04; // BCRL
-
-                // Conditional part first.
-                let cond_off = sink.cur_offset();
-                sink.use_label_at_offset(cond_off, taken, LabelUse::BranchRIL);
-                let inverted = &enc_ril_c(opcode, cond.invert().bits(), 0);
-                sink.add_cond_branch(cond_off, cond_off + 6, taken, inverted);
-                put(sink, &enc_ril_c(opcode, cond.bits(), 0));
-
-                // Unconditional part next.
-                let uncond_off = sink.cur_offset();
-                sink.use_label_at_offset(uncond_off, not_taken, LabelUse::BranchRIL);
-                sink.add_uncond_branch(uncond_off, uncond_off + 6, not_taken);
-                put(sink, &enc_ril_c(opcode, 15, 0));
-            }
-            &Inst::OneWayCondBr { target, cond } => {
-                let opcode = 0xc04; // BCRL
-                sink.use_label_at_offset(sink.cur_offset(), target, LabelUse::BranchRIL);
-                put(sink, &enc_ril_c(opcode, cond.bits(), 0));
-            }
-            &Inst::Nop0 => {}
-            &Inst::Nop2 => {
-                put(sink, &enc_e(0x0707));
-            }
-            &Inst::Debugtrap => {
-                put(sink, &enc_e(0x0001));
-            }
-            &Inst::Trap { trap_code } => {
-                put_with_trap(sink, &enc_e(0x0000), trap_code);
-            }
-            &Inst::TrapIf { cond, trap_code } => {
-                // We implement a TrapIf as a conditional branch into the middle
-                // of the branch (BRCL) instruction itself - those middle two bytes
-                // are zero, which matches the trap instruction itself.
-                let opcode = 0xc04; // BCRL
-                let enc = &enc_ril_c(opcode, cond.bits(), 2);
-                debug_assert!(enc.len() == 6 && enc[2] == 0 && enc[3] == 0);
-                // The trap must be placed on the last byte of the embedded trap
-                // instruction, so we need to emit the encoding in two parts.
-                put_with_trap(sink, &enc[0..4], trap_code);
-                put(sink, &enc[4..6]);
-            }
-            &Inst::JTSequence { ridx, ref targets } => {
-                let table_label = sink.get_label();
-
-                // This sequence is *one* instruction in the vcode, and is expanded only here at
-                // emission time, because we cannot allow the regalloc to insert spills/reloads in
-                // the middle; we depend on hardcoded PC-rel addressing below.
-
-                // Set temp register to address of jump table.
-                let rtmp = writable_spilltmp_reg();
-                let inst = Inst::LoadAddr {
-                    rd: rtmp,
-                    mem: MemArg::Label {
-                        target: table_label,
-                    },
-                };
-                inst.emit(sink, emit_info, state);
-
-                // Set temp to target address by adding the value of the jump table entry.
-                let inst = Inst::AluRX {
-                    alu_op: ALUOp::Add64Ext32,
-                    rd: rtmp,
-                    ri: rtmp.to_reg(),
-                    mem: MemArg::reg_plus_reg(rtmp.to_reg(), ridx, MemFlags::trusted()),
-                };
-                inst.emit(sink, emit_info, state);
-
-                // Branch to computed address. (`targets` here is only used for successor queries
-                // and is not needed for emission.)
-                let inst = Inst::IndirectBr {
-                    rn: rtmp.to_reg(),
-                    targets: vec![],
-                };
-                inst.emit(sink, emit_info, state);
-
-                // Emit jump table (table of 32-bit offsets).
-                sink.bind_label(table_label, &mut state.ctrl_plane);
-                let jt_off = sink.cur_offset();
-                for &target in targets.iter() {
-                    let word_off = sink.cur_offset();
-                    let off_into_table = word_off - jt_off;
-                    sink.use_label_at_offset(word_off, target, LabelUse::PCRel32);
-                    sink.put4(off_into_table.swap_bytes());
-                }
-
-                // Lowering produces an EmitIsland before using a JTSequence, so we can safely
-                // disable the worst-case-size check in this case.
-                start_off = sink.cur_offset();
-            }
-
-            Inst::StackProbeLoop {
-                probe_count,
-                guard_size,
-            } => {
-                // Emit the loop start label
-                let loop_start = sink.get_label();
-                sink.bind_label(loop_start, state.ctrl_plane_mut());
-
-                // aghi %r15, -GUARD_SIZE
-                let inst = Inst::AluRSImm16 {
-                    alu_op: ALUOp::Add64,
-                    rd: writable_stack_reg(),
-                    ri: stack_reg(),
-                    imm: -guard_size,
-                };
-                inst.emit(sink, emit_info, state);
-
-                // mvi 0(%r15), 0
-                let inst = Inst::StoreImm8 {
-                    imm: 0,
-                    mem: MemArg::reg(stack_reg(), MemFlags::trusted()),
-                };
-                inst.emit(sink, emit_info, state);
-
-                // brct PROBE_COUNT, LOOP_START
-                let opcode = 0xa76; // BRCT
-                sink.use_label_at_offset(sink.cur_offset(), loop_start, LabelUse::BranchRI);
-                put(sink, &enc_ri_b(opcode, probe_count.to_reg(), 0));
-            }
-
-            &Inst::Unwind { ref inst } => {
-                sink.add_unwind(inst.clone());
-            }
-
-            &Inst::DummyUse { .. } => {}
-        }
-
-        let end_off = sink.cur_offset();
-        debug_assert!((end_off - start_off) <= Inst::worst_case_size());
-
-        state.clear_post_insn();
-    }
-}