wasmtime 0.4.0 → 5.0.0

data/ext/cargo-vendor/cranelift-codegen-0.91.0/src/isa/aarch64/inst.isle

@@ -1,3749 +0,0 @@
-;; Instruction formats.
-(type MInst
-      (enum
-       ;; A no-op of zero size.
-       (Nop0)
-
-       ;; A no-op that is one instruction large.
-       (Nop4)
-
-       ;; An ALU operation with two register sources and a register destination.
-       (AluRRR
-        (alu_op ALUOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg))
-
-       ;; An ALU operation with three register sources and a register destination.
-       (AluRRRR
-        (alu_op ALUOp3)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (ra Reg))
-
-       ;; An ALU operation with a register source and an immediate-12 source, and a register
-       ;; destination.
-       (AluRRImm12
-        (alu_op ALUOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (imm12 Imm12))
-
-       ;; An ALU operation with a register source and an immediate-logic source, and a register destination.
-       (AluRRImmLogic
-        (alu_op ALUOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (imml ImmLogic))
-
-       ;; An ALU operation with a register source and an immediate-shiftamt source, and a register destination.
-       (AluRRImmShift
-        (alu_op ALUOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (immshift ImmShift))
-
-       ;; An ALU operation with two register sources, one of which can be shifted, and a register
-       ;; destination.
-       (AluRRRShift
-        (alu_op ALUOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (shiftop ShiftOpAndAmt))
-
-       ;; An ALU operation with two register sources, one of which can be {zero,sign}-extended and
-       ;; shifted, and a register destination.
-       (AluRRRExtend
-        (alu_op ALUOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (extendop ExtendOp))
-
-       ;; A bit op instruction with a single register source.
-       (BitRR
-        (op BitOp)
-        (size OperandSize)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; An unsigned (zero-extending) 8-bit load.
-       (ULoad8
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A signed (sign-extending) 8-bit load.
-       (SLoad8
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; An unsigned (zero-extending) 16-bit load.
-       (ULoad16
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A signed (sign-extending) 16-bit load.
-       (SLoad16
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; An unsigned (zero-extending) 32-bit load.
-       (ULoad32
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A signed (sign-extending) 32-bit load.
-       (SLoad32
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A 64-bit load.
-       (ULoad64
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; An 8-bit store.
-       (Store8
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A 16-bit store.
-       (Store16
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A 32-bit store.
-       (Store32
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A 64-bit store.
-       (Store64
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A store of a pair of registers.
-       (StoreP64
-        (rt Reg)
-        (rt2 Reg)
-        (mem PairAMode)
-        (flags MemFlags))
-
-       ;; A load of a pair of registers.
-       (LoadP64
-        (rt WritableReg)
-        (rt2 WritableReg)
-        (mem PairAMode)
-        (flags MemFlags))
-
-       ;; A MOV instruction. These are encoded as ORR's (AluRRR form).
-       ;; The 32-bit version zeroes the top 32 bits of the
-       ;; destination, which is effectively an alias for an unsigned
-       ;; 32-to-64-bit extension.
-       (Mov
-        (size OperandSize)
-        (rd WritableReg)
-        (rm Reg))
-
-       ;; Like `Move` but with a particular `PReg` source (for implementing CLIF
-       ;; instructions like `get_stack_pointer`).
-       (MovFromPReg
-        (rd WritableReg)
-        (rm PReg))
-
-       ;; Like `Move` but with a particular `PReg` destination (for
-       ;; implementing CLIF instructions like `set_pinned_reg`).
-       (MovToPReg
-        (rd PReg)
-        (rm Reg))
-
-       ;; A MOV[Z,N] with a 16-bit immediate.
-       (MovWide
-        (op MoveWideOp)
-        (rd WritableReg)
-        (imm MoveWideConst)
-        (size OperandSize))
-
-       ;; A MOVK with a 16-bit immediate. Modifies its register; we
-       ;; model this with a seprate input `rn` and output `rd` virtual
-       ;; register, with a regalloc constraint to tie them together.
-       (MovK
-        (rd WritableReg)
-        (rn Reg)
-        (imm MoveWideConst)
-        (size OperandSize))
-
-
-       ;; A sign- or zero-extend operation.
-       (Extend
-        (rd WritableReg)
-        (rn Reg)
-        (signed bool)
-        (from_bits u8)
-        (to_bits u8))
-
-       ;; A conditional-select operation.
-       (CSel
-        (rd WritableReg)
-        (cond Cond)
-        (rn Reg)
-        (rm Reg))
-
-       ;; A conditional-select negation operation.
-       (CSNeg
-        (rd WritableReg)
-        (cond Cond)
-        (rn Reg)
-        (rm Reg))
-
-       ;; A conditional-set operation.
-       (CSet
-        (rd WritableReg)
-        (cond Cond))
-
-       ;; A conditional-set-mask operation.
-       (CSetm
-        (rd WritableReg)
-        (cond Cond))
-
-       ;; A conditional comparison with a second register.
-       (CCmp
-        (size OperandSize)
-        (rn Reg)
-        (rm Reg)
-        (nzcv NZCV)
-        (cond Cond))
-
-       ;; A conditional comparison with an immediate.
-       (CCmpImm
-        (size OperandSize)
-        (rn Reg)
-        (imm UImm5)
-        (nzcv NZCV)
-        (cond Cond))
-
-       ;; A synthetic insn, which is a load-linked store-conditional loop, that has the overall
-       ;; effect of atomically modifying a memory location in a particular way.  Because we have
-       ;; no way to explain to the regalloc about earlyclobber registers, this instruction has
-       ;; completely fixed operand registers, and we rely on the RA's coalescing to remove copies
-       ;; in the surrounding code to the extent it can. Load- and store-exclusive instructions,
-       ;; with acquire-release semantics, are used to access memory. The operand conventions are:
-       ;;
-       ;; x25   (rd) address
-       ;; x26   (rd) second operand for `op`
-       ;; x27   (wr) old value
-       ;; x24   (wr) scratch reg; value afterwards has no meaning
-       ;; x28   (wr) scratch reg; value afterwards has no meaning
-       (AtomicRMWLoop
-        (ty Type) ;; I8, I16, I32 or I64
-        (op AtomicRMWLoopOp)
-        (flags MemFlags)
-        (addr Reg)
-        (operand Reg)
-        (oldval WritableReg)
-        (scratch1 WritableReg)
-        (scratch2 WritableReg))
-
-       ;; Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked
-       ;; store-conditional loop, with acquire-release semantics.
-       ;; Note that the operand conventions, although very similar to AtomicRMWLoop, are different:
-       ;;
-       ;; x25   (rd) address
-       ;; x26   (rd) expected value
-       ;; x28   (rd) replacement value
-       ;; x27   (wr) old value
-       ;; x24   (wr) scratch reg; value afterwards has no meaning
-       (AtomicCASLoop
-        (ty Type) ;; I8, I16, I32 or I64
-        (flags MemFlags)
-        (addr Reg)
-        (expected Reg)
-        (replacement Reg)
-        (oldval WritableReg)
-        (scratch WritableReg))
-
-       ;; An atomic read-modify-write operation. These instructions require the
-       ;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
-       ;; acquire-release semantics.
-       (AtomicRMW
-         (op AtomicRMWOp)
-         (rs Reg)
-         (rt WritableReg)
-         (rn Reg)
-         (ty Type)
-         (flags MemFlags))
-
-       ;; An atomic compare-and-swap operation. These instructions require the
-       ;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
-       ;; acquire-release semantics.
-       (AtomicCAS
-         ;; `rd` is really `rs` in the encoded instruction (so `rd` == `rs`); we separate
-         ;; them here to have separate use and def vregs for regalloc.
-         (rd WritableReg)
-         (rs Reg)
-         (rt Reg)
-         (rn Reg)
-         (ty Type)
-         (flags MemFlags))
-
-       ;; Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put
-       ;; it in `rn`, optionally zero-extending to fill a word or double word result.
-       ;; This instruction is sequentially consistent.
-       (LoadAcquire
-        (access_ty Type) ;; I8, I16, I32 or I64
-        (rt WritableReg)
-        (rn Reg)
-        (flags MemFlags))
-
-       ;; Write the lowest `ty` bits of `rt` to address `rn`.
-       ;; This instruction is sequentially consistent.
-       (StoreRelease
-        (access_ty Type) ;; I8, I16, I32 or I64
-        (rt Reg)
-        (rn Reg)
-        (flags MemFlags))
-
-       ;; A memory fence.  This must provide ordering to ensure that, at a minimum, neither loads
-       ;; nor stores may move forwards or backwards across the fence.  Currently emitted as "dmb
-       ;; ish".  This instruction is sequentially consistent.
-       (Fence)
-
-       ;; Consumption of speculative data barrier.
-       (Csdb)
-
-       ;; FPU move. Note that this is distinct from a vector-register
-       ;; move; moving just 64 bits seems to be significantly faster.
-       (FpuMove64
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; Vector register move.
-       (FpuMove128
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; Move to scalar from a vector element.
-       (FpuMoveFromVec
-        (rd WritableReg)
-        (rn Reg)
-        (idx u8)
-        (size VectorSize))
-
-       ;; Zero-extend a SIMD & FP scalar to the full width of a vector register.
-       ;; 16-bit scalars require half-precision floating-point support (FEAT_FP16).
-       (FpuExtend
-        (rd WritableReg)
-        (rn Reg)
-        (size ScalarSize))
-
-       ;; 1-op FPU instruction.
-       (FpuRR
-        (fpu_op FPUOp1)
-        (size ScalarSize)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; 2-op FPU instruction.
-       (FpuRRR
-        (fpu_op FPUOp2)
-        (size ScalarSize)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg))
-
-       (FpuRRI
-        (fpu_op FPUOpRI)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; Variant of FpuRRI that modifies its `rd`, and so we name the
-       ;; input state `ri` (for "input") and constrain the two
-       ;; together.
-       (FpuRRIMod
-        (fpu_op FPUOpRIMod)
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg))
-
-
-       ;; 3-op FPU instruction.
-       ;; 16-bit scalars require half-precision floating-point support (FEAT_FP16).
-       (FpuRRRR
-        (fpu_op FPUOp3)
-        (size ScalarSize)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (ra Reg))
-
-       ;; FPU comparison.
-       (FpuCmp
-        (size ScalarSize)
-        (rn Reg)
-        (rm Reg))
-
-       ;; Floating-point load, single-precision (32 bit).
-       (FpuLoad32
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; Floating-point store, single-precision (32 bit).
-       (FpuStore32
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; Floating-point load, double-precision (64 bit).
-       (FpuLoad64
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; Floating-point store, double-precision (64 bit).
-       (FpuStore64
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; Floating-point/vector load, 128 bit.
-       (FpuLoad128
-        (rd WritableReg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; Floating-point/vector store, 128 bit.
-       (FpuStore128
-        (rd Reg)
-        (mem AMode)
-        (flags MemFlags))
-
-       ;; A load of a pair of floating-point registers, double precision (64-bit).
-       (FpuLoadP64
-        (rt WritableReg)
-        (rt2 WritableReg)
-        (mem PairAMode)
-        (flags MemFlags))
-
-       ;; A store of a pair of floating-point registers, double precision (64-bit).
-       (FpuStoreP64
-        (rt Reg)
-        (rt2 Reg)
-        (mem PairAMode)
-        (flags MemFlags))
-
-       ;; A load of a pair of floating-point registers, 128-bit.
-       (FpuLoadP128
-        (rt WritableReg)
-        (rt2 WritableReg)
-        (mem PairAMode)
-        (flags MemFlags))
-
-       ;; A store of a pair of floating-point registers, 128-bit.
-       (FpuStoreP128
-        (rt Reg)
-        (rt2 Reg)
-        (mem PairAMode)
-        (flags MemFlags))
-
-       (LoadFpuConst64
-        (rd WritableReg)
-        (const_data u64))
-
-       (LoadFpuConst128
-        (rd WritableReg)
-        (const_data u128))
-
-       ;; Conversion: FP -> integer.
-       (FpuToInt
-        (op FpuToIntOp)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; Conversion: integer -> FP.
-       (IntToFpu
-        (op IntToFpuOp)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; FP conditional select, 32 bit.
-       (FpuCSel32
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (cond Cond))
-
-       ;; FP conditional select, 64 bit.
-       (FpuCSel64
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (cond Cond))
-
-       ;; Round to integer.
-       (FpuRound
-        (op FpuRoundMode)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; Move from a GPR to a vector register.  The scalar value is parked in the lowest lane
-       ;; of the destination, and all other lanes are zeroed out.  Currently only 32- and 64-bit
-       ;; transactions are supported.
-       (MovToFpu
-        (rd WritableReg)
-        (rn Reg)
-        (size ScalarSize))
-
-       ;; Loads a floating-point immediate.
-       (FpuMoveFPImm
-        (rd WritableReg)
-        (imm ASIMDFPModImm)
-        (size ScalarSize))
-
-       ;; Move to a vector element from a GPR.
-       (MovToVec
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (idx u8)
-        (size VectorSize))
-
-       ;; Unsigned move from a vector element to a GPR.
-       (MovFromVec
-        (rd WritableReg)
-        (rn Reg)
-        (idx u8)
-        (size ScalarSize))
-
-       ;; Signed move from a vector element to a GPR.
-       (MovFromVecSigned
-        (rd WritableReg)
-        (rn Reg)
-        (idx u8)
-        (size VectorSize)
-        (scalar_size OperandSize))
-
-       ;; Duplicate general-purpose register to vector.
-       (VecDup
-        (rd WritableReg)
-        (rn Reg)
-        (size VectorSize))
-
-       ;; Duplicate scalar to vector.
-       (VecDupFromFpu
-        (rd WritableReg)
-        (rn Reg)
-        (size VectorSize))
-
-       ;; Duplicate FP immediate to vector.
-       (VecDupFPImm
-        (rd WritableReg)
-        (imm ASIMDFPModImm)
-        (size VectorSize))
-
-       ;; Duplicate immediate to vector.
-       (VecDupImm
-        (rd WritableReg)
-        (imm ASIMDMovModImm)
-        (invert bool)
-        (size VectorSize))
-
-       ;; Vector extend.
-       (VecExtend
-        (t VecExtendOp)
-        (rd WritableReg)
-        (rn Reg)
-        (high_half bool)
-        (lane_size ScalarSize))
-
-       ;; Move vector element to another vector element.
-       (VecMovElement
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (dest_idx u8)
-        (src_idx u8)
-        (size VectorSize))
-
-       ;; Vector widening operation.
-       (VecRRLong
-        (op VecRRLongOp)
-        (rd WritableReg)
-        (rn Reg)
-        (high_half bool))
-
-       ;; Vector narrowing operation -- low half.
-       (VecRRNarrowLow
-        (op VecRRNarrowOp)
-        (rd WritableReg)
-        (rn Reg)
-        (lane_size ScalarSize))
-
-       ;; Vector narrowing operation -- high half.
-       (VecRRNarrowHigh
-        (op VecRRNarrowOp)
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (lane_size ScalarSize))
-
-       ;; 1-operand vector instruction that operates on a pair of elements.
-       (VecRRPair
-        (op VecPairOp)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; 2-operand vector instruction that produces a result with twice the
-       ;; lane width and half the number of lanes.
-       (VecRRRLong
-        (alu_op VecRRRLongOp)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (high_half bool))
-
-       ;; 2-operand vector instruction that produces a result with
-       ;; twice the lane width and half the number of lanes. Variant
-       ;; that modifies `rd` (so takes its initial state as `ri`).
-       (VecRRRLongMod
-        (alu_op VecRRRLongModOp)
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (rm Reg)
-        (high_half bool))
-
-       ;; 1-operand vector instruction that extends elements of the input
-       ;; register and operates on a pair of elements. The output lane width
-       ;; is double that of the input.
-       (VecRRPairLong
-        (op VecRRPairLongOp)
-        (rd WritableReg)
-        (rn Reg))
-
-       ;; A vector ALU op.
-       (VecRRR
-        (alu_op VecALUOp)
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (size VectorSize))
-
-       ;; A vector ALU op modifying a source register.
-       (VecRRRMod
-        (alu_op VecALUModOp)
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (rm Reg)
-        (size VectorSize))
-
-       ;; Vector two register miscellaneous instruction.
-       (VecMisc
-        (op VecMisc2)
-        (rd WritableReg)
-        (rn Reg)
-        (size VectorSize))
-
-       ;; Vector instruction across lanes.
-       (VecLanes
-        (op VecLanesOp)
-        (rd WritableReg)
-        (rn Reg)
-        (size VectorSize))
-
-       ;; Vector shift by immediate Shift Left (immediate), Unsigned Shift Right (immediate)
-       ;; Signed Shift Right (immediate).  These are somewhat unusual in that, for right shifts,
-       ;; the allowed range of `imm` values is 1 to lane-size-in-bits, inclusive.  A zero
-       ;; right-shift cannot be encoded.  Left shifts are "normal", though, having valid `imm`
-       ;; values from 0 to lane-size-in-bits - 1 inclusive.
-       (VecShiftImm
-        (op VecShiftImmOp)
-        (rd WritableReg)
-        (rn Reg)
-        (size VectorSize)
-        (imm u8))
-
-       ;; Destructive vector shift by immediate.
-       (VecShiftImmMod
-        (op VecShiftImmModOp)
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (size VectorSize)
-        (imm u8))
-
-       ;; Vector extract - create a new vector, being the concatenation of the lowest `imm4` bytes
-       ;; of `rm` followed by the uppermost `16 - imm4` bytes of `rn`.
-       (VecExtract
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (imm4 u8))
-
-       ;; Table vector lookup - single register table. The table
-       ;; consists of 8-bit elements and is stored in `rn`, while `rm`
-       ;; contains 8-bit element indices. This variant emits `TBL`,
-       ;; which sets elements that correspond to out-of-range indices
-       ;; (greater than 15) to 0.
-       (VecTbl
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg))
-
-       ;; Table vector lookup - single register table. The table
-       ;; consists of 8-bit elements and is stored in `rn`, while `rm`
-       ;; contains 8-bit element indices. This variant emits `TBX`,
-       ;; which leaves elements that correspond to out-of-range indices
-       ;; (greater than 15) unmodified. Hence, it takes an input vreg in
-       ;; `ri` that is constrained to the same allocation as `rd`.
-       (VecTblExt
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (rm Reg))
-
-       ;; Table vector lookup - two register table. The table consists
-       ;; of 8-bit elements and is stored in `rn` and `rn2`, while
-       ;; `rm` contains 8-bit element indices. The table registers
-       ;; `rn` and `rn2` must have consecutive numbers modulo 32, that
-       ;; is v31 and v0 (in that order) are consecutive registers.
-       ;; This variant emits `TBL`, which sets out-of-range results to
-       ;; 0.
-       (VecTbl2
-        (rd WritableReg)
-        (rn Reg)
-        (rn2 Reg)
-        (rm Reg))
-
-       ;; Table vector lookup - two register table. The table consists
-       ;; of 8-bit elements and is stored in `rn` and `rn2`, while
-       ;; `rm` contains 8-bit element indices. The table registers
-       ;; `rn` and `rn2` must have consecutive numbers modulo 32, that
-       ;; is v31 and v0 (in that order) are consecutive registers.
-       ;; This variant emits `TBX`, which leaves out-of-range results
-       ;; unmodified, hence takes the initial state of the result
-       ;; register in vreg `ri`.
-       (VecTbl2Ext
-        (rd WritableReg)
-        (ri Reg)
-        (rn Reg)
-        (rn2 Reg)
-        (rm Reg))
-
-       ;; Load an element and replicate to all lanes of a vector.
-       (VecLoadReplicate
-        (rd WritableReg)
-        (rn Reg)
-        (size VectorSize)
-        (flags MemFlags))
-
-       ;; Vector conditional select, 128 bit.  A synthetic instruction, which generates a 4-insn
-       ;; control-flow diamond.
-       (VecCSel
-        (rd WritableReg)
-        (rn Reg)
-        (rm Reg)
-        (cond Cond))
-
-       ;; Move to the NZCV flags (actually a `MSR NZCV, Xn` insn).
-       (MovToNZCV
-        (rn Reg))
-
-       ;; Move from the NZCV flags (actually a `MRS Xn, NZCV` insn).
-       (MovFromNZCV
-        (rd WritableReg))
-
-       ;; A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation
-       ;; of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the
-       ;; code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit
-       ;; target.
-       (Call
-        (info BoxCallInfo))
-
-       ;; A machine indirect-call instruction.
-       (CallInd
-        (info BoxCallIndInfo))
-
-       ;; A pseudo-instruction that captures register arguments in vregs.
-       (Args
-        (args VecArgPair))
-
-       ;; ---- branches (exactly one must appear at end of BB) ----
-
-       ;; A machine return instruction.
-       (Ret
-        (rets VecRetPair))
-
-       ;; A machine return instruction with pointer authentication using SP as the
-       ;; modifier. This instruction requires pointer authentication support
-       ;; (FEAT_PAuth) unless `is_hint` is true, in which case it is equivalent to
-       ;; the combination of a no-op and a return instruction on platforms without
-       ;; the relevant support.
-       (AuthenticatedRet
-        (key APIKey)
-        (is_hint bool)
-        (rets VecRetPair))
-
-       ;; An unconditional branch.
-       (Jump
-        (dest BranchTarget))
-
-       ;; A conditional branch. Contains two targets; at emission time, both are emitted, but
-       ;; the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the
-       ;; choice of taken/not_taken (inverting the branch polarity as needed) based on the
-       ;; fallthrough at the time of lowering.
-       (CondBr
-        (taken BranchTarget)
-        (not_taken BranchTarget)
-        (kind CondBrKind))
-
-       ;; A conditional trap: execute a `udf` if the condition is true. This is
-       ;; one VCode instruction because it uses embedded control flow; it is
-       ;; logically a single-in, single-out region, but needs to appear as one
-       ;; unit to the register allocator.
-       ;;
-       ;; The `CondBrKind` gives the conditional-branch condition that will
-       ;; *execute* the embedded `Inst`. (In the emitted code, we use the inverse
-       ;; of this condition in a branch that skips the trap instruction.)
-       (TrapIf
-        (kind CondBrKind)
-        (trap_code TrapCode))
-
-       ;; An indirect branch through a register, augmented with set of all
-       ;; possible successors.
-       (IndirectBr
-        (rn Reg)
-        (targets VecMachLabel))
-
-       ;; A "break" instruction, used for e.g. traps and debug breakpoints.
-       (Brk)
-
-       ;; An instruction guaranteed to always be undefined and to trigger an illegal instruction at
-       ;; runtime.
-       (Udf
-        (trap_code TrapCode))
-
-       ;; Compute the address (using a PC-relative offset) of a memory location, using the `ADR`
-       ;; instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is
-       ;; only used for now in fixed lowering sequences with hardcoded offsets. In the future we may
-       ;; need full `MemLabel` support.
-       (Adr
-        (rd WritableReg)
-        ;; Offset in range -2^20 .. 2^20.
-        (off i32))
-
-       ;; Raw 32-bit word, used for inline constants and jump-table entries.
-       (Word4
-        (data u32))
-
-       ;; Raw 64-bit word, used for inline constants.
-       (Word8
-        (data u64))
-
-       ;; Jump-table sequence, as one compound instruction (see note in lower_inst.rs for rationale).
-       (JTSequence
-        (info BoxJTSequenceInfo)
-        (ridx Reg)
-        (rtmp1 WritableReg)
-        (rtmp2 WritableReg))
-
-       ;; Load an inline symbol reference.
-       (LoadExtName
-        (rd WritableReg)
-        (name BoxExternalName)
-        (offset i64))
-
-       ;; Load address referenced by `mem` into `rd`.
-       (LoadAddr
-        (rd WritableReg)
-        (mem AMode))
-
-       ;; Pointer authentication code for instruction address with modifier in SP;
-       ;; equivalent to a no-op if Pointer authentication (FEAT_PAuth) is not
-       ;; supported.
-       (Pacisp
-        (key APIKey))
-
-       ;; Strip pointer authentication code from instruction address in LR;
-       ;; equivalent to a no-op if Pointer authentication (FEAT_PAuth) is not
-       ;; supported.
-       (Xpaclri)
-
-       ;; Branch target identification; equivalent to a no-op if Branch Target
-       ;; Identification (FEAT_BTI) is not supported.
-       (Bti
-        (targets BranchTargetType))
-
-       ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. This
-       ;; controls how AMode::NominalSPOffset args are lowered.
-       (VirtualSPOffsetAdj
-        (offset i64))
-
-       ;; Meta-insn, no-op in generated code: emit constant/branch veneer island
-       ;; at this point (with a guard jump around it) if less than the needed
-       ;; space is available before the next branch deadline. See the `MachBuffer`
-       ;; implementation in `machinst/buffer.rs` for the overall algorithm. In
-       ;; brief, we retain a set of "pending/unresolved label references" from
-       ;; branches as we scan forward through instructions to emit machine code;
-       ;; if we notice we're about to go out of range on an unresolved reference,
-       ;; we stop, emit a bunch of "veneers" (branches in a form that has a longer
-       ;; range, e.g. a 26-bit-offset unconditional jump), and point the original
-       ;; label references to those. This is an "island" because it comes in the
-       ;; middle of the code.
-       ;;
-       ;; This meta-instruction is a necessary part of the logic that determines
-       ;; where to place islands. Ordinarily, we want to place them between basic
-       ;; blocks, so we compute the worst-case size of each block, and emit the
-       ;; island before starting a block if we would exceed a deadline before the
-       ;; end of the block. However, some sequences (such as an inline jumptable)
-       ;; are variable-length and not accounted for by this logic; so these
-       ;; lowered sequences include an `EmitIsland` to trigger island generation
-       ;; where necessary.
-       (EmitIsland
-        ;; The needed space before the next deadline.
-        (needed_space CodeOffset))
-
-       ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0.
-       (ElfTlsGetAddr
-        (symbol ExternalName)
-        (rd WritableReg))
-
-       ;; An unwind pseudo-instruction.
-       (Unwind
-        (inst UnwindInst))
-
-       ;; A dummy use, useful to keep a value alive.
-       (DummyUse
-        (reg Reg))
-
-       ;; Emits an inline stack probe loop.
-       ;;
-       ;; Note that this is emitted post-regalloc so `start` and `end` can be
-       ;; temporary registers such as the spilltmp and tmp2 registers. This also
-       ;; means that the internal codegen can't use these registers.
-       (StackProbeLoop (start WritableReg)
-                       (end Reg)
-                       (step Imm12))))
-
-;; An ALU operation. This can be paired with several instruction formats
-;; below (see `Inst`) in any combination.
-(type ALUOp
-  (enum
-    (Add)
-    (Sub)
-    (Orr)
-    (OrrNot)
-    (And)
-    (AndS)
-    (AndNot)
-    ;; XOR (AArch64 calls this "EOR")
-    (Eor)
-    ;; XNOR (AArch64 calls this "EOR-NOT")
-    (EorNot)
-    ;; Add, setting flags
-    (AddS)
-    ;; Sub, setting flags
-    (SubS)
-    ;; Signed multiply, high-word result
-    (SMulH)
-    ;; Unsigned multiply, high-word result
-    (UMulH)
-    (SDiv)
-    (UDiv)
-    (RotR)
-    (Lsr)
-    (Asr)
-    (Lsl)
-    ;; Add with carry
-    (Adc)
-    ;; Add with carry, settings flags
-    (AdcS)
-    ;; Subtract with carry
-    (Sbc)
-    ;; Subtract with carry, settings flags
-    (SbcS)
-))
-
-;; An ALU operation with three arguments.
-(type ALUOp3
-  (enum
-    ;; Multiply-add
-    (MAdd)
-    ;; Multiply-sub
-    (MSub)
-))
-
-(type MoveWideOp
-  (enum
-    (MovZ)
-    (MovN)
-))
-
-(type UImm5 (primitive UImm5))
-(type Imm12 (primitive Imm12))
-(type ImmLogic (primitive ImmLogic))
-(type ImmShift (primitive ImmShift))
-(type ShiftOpAndAmt (primitive ShiftOpAndAmt))
-(type MoveWideConst (primitive MoveWideConst))
-(type NZCV (primitive NZCV))
-(type ASIMDFPModImm (primitive ASIMDFPModImm))
-(type ASIMDMovModImm (primitive ASIMDMovModImm))
-
-(type BoxCallInfo (primitive BoxCallInfo))
-(type BoxCallIndInfo (primitive BoxCallIndInfo))
-(type CondBrKind (primitive CondBrKind))
-(type BranchTarget (primitive BranchTarget))
-(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
-(type CodeOffset (primitive CodeOffset))
-(type VecMachLabel extern (enum))
-
-(type ExtendOp extern
-  (enum
-    (UXTB)
-    (UXTH)
-    (UXTW)
-    (UXTX)
-    (SXTB)
-    (SXTH)
-    (SXTW)
-    (SXTX)
-))
-
-;; An operation on the bits of a register. This can be paired with several instruction formats
-;; below (see `Inst`) in any combination.
-(type BitOp
-  (enum
-    ;; Bit reverse
-    (RBit)
-    (Clz)
-    (Cls)
-    ;; Byte reverse
-    (Rev16)
-    (Rev32)
-    (Rev64)
-))
-
-(type MemLabel extern (enum))
-(type SImm9 extern (enum))
-(type UImm12Scaled extern (enum))
-
-;; An addressing mode specified for a load/store operation.
-(type AMode
-      (enum
-        ;;
-        ;; Real ARM64 addressing modes:
-        ;;
-        ;; "post-indexed" mode as per AArch64 docs: postincrement reg after
-        ;; address computation.
-        ;; Specialized here to SP so we don't have to emit regalloc metadata.
-        (SPPostIndexed
-         (simm9 SImm9))
-
-        ;; "pre-indexed" mode as per AArch64 docs: preincrement reg before
-        ;; address computation.
-        ;; Specialized here to SP so we don't have to emit regalloc metadata.
-        (SPPreIndexed
-         (simm9 SImm9))
-
-        ;; N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to
-        ;; what the ISA calls the "register offset" addressing mode. We split
-        ;; out several options here for more ergonomic codegen.
-        ;;
-        ;; Register plus register offset.
-        (RegReg
-         (rn Reg)
-         (rm Reg))
-
-        ;; Register plus register offset, scaled by type's size.
-        (RegScaled
-         (rn Reg)
-         (rm Reg)
-         (ty Type))
-
-        ;; Register plus register offset, scaled by type's size, with index
-        ;; sign- or zero-extended first.
-        (RegScaledExtended
-         (rn Reg)
-         (rm Reg)
-         (ty Type)
-         (extendop ExtendOp))
-
-        ;; Register plus register offset, with index sign- or zero-extended
-        ;; first.
-        (RegExtended
-         (rn Reg)
-         (rm Reg)
-         (extendop ExtendOp))
-
-        ;; Unscaled signed 9-bit immediate offset from reg.
-        (Unscaled
-         (rn Reg)
-         (simm9 SImm9))
-
-        ;; Scaled (by size of a type) unsigned 12-bit immediate offset from reg.
-        (UnsignedOffset
-         (rn Reg)
-         (uimm12 UImm12Scaled))
-
-        ;; virtual addressing modes that are lowered at emission time:
-        ;;
-        ;; Reference to a "label": e.g., a symbol.
-        (Label
-         (label MemLabel))
-
-        ;; Arbitrary offset from a register. Converted to generation of large
-        ;; offsets with multiple instructions as necessary during code emission.
-        (RegOffset
-         (rn Reg)
-         (off i64)
-         (ty Type))
-
-        ;; Offset from the stack pointer.
-        (SPOffset
-         (off i64)
-         (ty Type))
-
-        ;; Offset from the frame pointer.
-        (FPOffset
-         (off i64)
-         (ty Type))
-
-        ;; Offset from the "nominal stack pointer", which is where the real SP is
-        ;; just after stack and spill slots are allocated in the function prologue.
-        ;; At emission time, this is converted to `SPOffset` with a fixup added to
-        ;; the offset constant. The fixup is a running value that is tracked as
-        ;; emission iterates through instructions in linear order, and can be
-        ;; adjusted up and down with [Inst::VirtualSPOffsetAdj].
-        ;;
-        ;; The standard ABI is in charge of handling this (by emitting the
-        ;; adjustment meta-instructions). It maintains the invariant that "nominal
-        ;; SP" is where the actual SP is after the function prologue and before
-        ;; clobber pushes. See the diagram in the documentation for
-        ;; [crate::isa::aarch64::abi](the ABI module) for more details.
-        (NominalSPOffset
-         (off i64)
-         (ty Type))))
-
-(type PairAMode extern (enum))
-(type FPUOpRI extern (enum))
-(type FPUOpRIMod extern (enum))
-
-(type OperandSize extern
-      (enum Size32
-            Size64))
-
-;; Helper for calculating the `OperandSize` corresponding to a type
-(decl operand_size (Type) OperandSize)
-(rule 1 (operand_size (fits_in_32 _ty)) (OperandSize.Size32))
-(rule (operand_size (fits_in_64 _ty)) (OperandSize.Size64))
-
-(type ScalarSize extern
-      (enum Size8
-            Size16
-            Size32
-            Size64
-            Size128))
-
-;; Helper for calculating the `ScalarSize` corresponding to a type
-(decl scalar_size (Type) ScalarSize)
-
-(rule (scalar_size $I8) (ScalarSize.Size8))
-(rule (scalar_size $I16) (ScalarSize.Size16))
-(rule (scalar_size $I32) (ScalarSize.Size32))
-(rule (scalar_size $I64) (ScalarSize.Size64))
-(rule (scalar_size $I128) (ScalarSize.Size128))
-
-(rule (scalar_size $F32) (ScalarSize.Size32))
-(rule (scalar_size $F64) (ScalarSize.Size64))
-
-;; Helper for calculating the `ScalarSize` lane type from vector type
-(decl lane_size (Type) ScalarSize)
-(rule 1 (lane_size (multi_lane 8 _)) (ScalarSize.Size8))
-(rule 1 (lane_size (multi_lane 16 _)) (ScalarSize.Size16))
-(rule 1 (lane_size (multi_lane 32 _)) (ScalarSize.Size32))
-(rule 1 (lane_size (multi_lane 64 _)) (ScalarSize.Size64))
-(rule (lane_size (dynamic_lane 8 _)) (ScalarSize.Size8))
-(rule (lane_size (dynamic_lane 16 _)) (ScalarSize.Size16))
-(rule (lane_size (dynamic_lane 32 _)) (ScalarSize.Size32))
-(rule (lane_size (dynamic_lane 64 _)) (ScalarSize.Size64))
-
-(type Cond extern
-  (enum
-    (Eq)
-    (Ne)
-    (Hs)
-    (Lo)
-    (Mi)
-    (Pl)
-    (Vs)
-    (Vc)
-    (Hi)
-    (Ls)
-    (Ge)
-    (Lt)
-    (Gt)
-    (Le)
-    (Al)
-    (Nv)
-))
-
-(type VectorSize extern
-  (enum
-    (Size8x8)
-    (Size8x16)
-    (Size16x4)
-    (Size16x8)
-    (Size32x2)
-    (Size32x4)
-    (Size64x2)
-))
-
-;; Helper for calculating the `VectorSize` corresponding to a type
-(decl vector_size (Type) VectorSize)
-(rule 1 (vector_size (multi_lane 8 8)) (VectorSize.Size8x8))
-(rule 1 (vector_size (multi_lane 8 16)) (VectorSize.Size8x16))
-(rule 1 (vector_size (multi_lane 16 4)) (VectorSize.Size16x4))
-(rule 1 (vector_size (multi_lane 16 8)) (VectorSize.Size16x8))
-(rule 1 (vector_size (multi_lane 32 2)) (VectorSize.Size32x2))
-(rule 1 (vector_size (multi_lane 32 4)) (VectorSize.Size32x4))
-(rule 1 (vector_size (multi_lane 64 2)) (VectorSize.Size64x2))
-(rule (vector_size (dynamic_lane 8 8)) (VectorSize.Size8x8))
-(rule (vector_size (dynamic_lane 8 16)) (VectorSize.Size8x16))
-(rule (vector_size (dynamic_lane 16 4)) (VectorSize.Size16x4))
-(rule (vector_size (dynamic_lane 16 8)) (VectorSize.Size16x8))
-(rule (vector_size (dynamic_lane 32 2)) (VectorSize.Size32x2))
-(rule (vector_size (dynamic_lane 32 4)) (VectorSize.Size32x4))
-(rule (vector_size (dynamic_lane 64 2)) (VectorSize.Size64x2))
-
-;; A floating-point unit (FPU) operation with one arg.
-(type FPUOp1
-  (enum
-    (Abs)
-    (Neg)
-    (Sqrt)
-    (Cvt32To64)
-    (Cvt64To32)
-))
-
-;; A floating-point unit (FPU) operation with two args.
-(type FPUOp2
-  (enum
-    (Add)
-    (Sub)
-    (Mul)
-    (Div)
-    (Max)
-    (Min)
-))
-
-;; A floating-point unit (FPU) operation with three args.
-(type FPUOp3
-  (enum
-    (MAdd)
-))
-
-;; A conversion from an FP to an integer value.
-(type FpuToIntOp
-  (enum
-    (F32ToU32)
-    (F32ToI32)
-    (F32ToU64)
-    (F32ToI64)
-    (F64ToU32)
-    (F64ToI32)
-    (F64ToU64)
-    (F64ToI64)
-))
-
-;; A conversion from an integer to an FP value.
-(type IntToFpuOp
-  (enum
-    (U32ToF32)
-    (I32ToF32)
-    (U32ToF64)
-    (I32ToF64)
-    (U64ToF32)
-    (I64ToF32)
-    (U64ToF64)
-    (I64ToF64)
-))
-
-;; Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to
-;; nearest, and for 32- or 64-bit FP values.
-(type FpuRoundMode
-  (enum
-    (Minus32)
-    (Minus64)
-    (Plus32)
-    (Plus64)
-    (Zero32)
-    (Zero64)
-    (Nearest32)
-    (Nearest64)
-))
-
-;; Type of vector element extensions.
-(type VecExtendOp
-  (enum
-    ;; Signed extension
-    (Sxtl)
-    ;; Unsigned extension
-    (Uxtl)
-))
-
-;; A vector ALU operation.
-(type VecALUOp
-  (enum
-    ;; Signed saturating add
-    (Sqadd)
-    ;; Unsigned saturating add
-    (Uqadd)
-    ;; Signed saturating subtract
-    (Sqsub)
-    ;; Unsigned saturating subtract
-    (Uqsub)
-    ;; Compare bitwise equal
-    (Cmeq)
-    ;; Compare signed greater than or equal
-    (Cmge)
-    ;; Compare signed greater than
-    (Cmgt)
-    ;; Compare unsigned higher
-    (Cmhs)
-    ;; Compare unsigned higher or same
-    (Cmhi)
-    ;; Floating-point compare equal
-    (Fcmeq)
-    ;; Floating-point compare greater than
-    (Fcmgt)
-    ;; Floating-point compare greater than or equal
-    (Fcmge)
-    ;; Bitwise and
-    (And)
-    ;; Bitwise bit clear
-    (Bic)
-    ;; Bitwise inclusive or
-    (Orr)
-    ;; Bitwise exclusive or
-    (Eor)
-    ;; Unsigned maximum pairwise
-    (Umaxp)
-    ;; Add
-    (Add)
-    ;; Subtract
-    (Sub)
-    ;; Multiply
-    (Mul)
-    ;; Signed shift left
-    (Sshl)
-    ;; Unsigned shift left
-    (Ushl)
-    ;; Unsigned minimum
-    (Umin)
-    ;; Signed minimum
-    (Smin)
-    ;; Unsigned maximum
-    (Umax)
-    ;; Signed maximum
-    (Smax)
-    ;; Unsigned rounding halving add
-    (Urhadd)
-    ;; Floating-point add
-    (Fadd)
-    ;; Floating-point subtract
-    (Fsub)
-    ;; Floating-point divide
-    (Fdiv)
-    ;; Floating-point maximum
-    (Fmax)
-    ;; Floating-point minimum
-    (Fmin)
-    ;; Floating-point multiply
-    (Fmul)
-    ;; Add pairwise
-    (Addp)
-    ;; Zip vectors (primary) [meaning, high halves]
-    (Zip1)
-    ;; Signed saturating rounding doubling multiply returning high half
-    (Sqrdmulh)
-))
-
-;; A Vector ALU operation which modifies a source register.
-(type VecALUModOp
-  (enum
-    ;; Bitwise select
-    (Bsl)
-    ;; Floating-point fused multiply-add vectors
-    (Fmla)
-))
-
-;; A Vector miscellaneous operation with two registers.
-(type VecMisc2
-  (enum
-    ;; Bitwise NOT
-    (Not)
-    ;; Negate
-    (Neg)
-    ;; Absolute value
-    (Abs)
-    ;; Floating-point absolute value
-    (Fabs)
-    ;; Floating-point negate
-    (Fneg)
-    ;; Floating-point square root
-    (Fsqrt)
-    ;; Reverse elements in 64-bit doublewords
-    (Rev64)
-    ;; Floating-point convert to signed integer, rounding toward zero
-    (Fcvtzs)
-    ;; Floating-point convert to unsigned integer, rounding toward zero
-    (Fcvtzu)
-    ;; Signed integer convert to floating-point
-    (Scvtf)
-    ;; Unsigned integer convert to floating-point
-    (Ucvtf)
-    ;; Floating point round to integral, rounding towards nearest
-    (Frintn)
-    ;; Floating point round to integral, rounding towards zero
-    (Frintz)
-    ;; Floating point round to integral, rounding towards minus infinity
-    (Frintm)
-    ;; Floating point round to integral, rounding towards plus infinity
-    (Frintp)
-    ;; Population count per byte
-    (Cnt)
-    ;; Compare bitwise equal to 0
-    (Cmeq0)
-    ;; Compare signed greater than or equal to 0
-    (Cmge0)
-    ;; Compare signed greater than 0
-    (Cmgt0)
-    ;; Compare signed less than or equal to 0
-    (Cmle0)
-    ;; Compare signed less than 0
-    (Cmlt0)
-    ;; Floating point compare equal to 0
-    (Fcmeq0)
-    ;; Floating point compare greater than or equal to 0
-    (Fcmge0)
-    ;; Floating point compare greater than 0
-    (Fcmgt0)
-    ;; Floating point compare less than or equal to 0
-    (Fcmle0)
-    ;; Floating point compare less than 0
-    (Fcmlt0)
-))
-
-;; A vector widening operation with one argument.
-(type VecRRLongOp
-  (enum
-    ;; Floating-point convert to higher precision long, 16-bit elements
-    (Fcvtl16)
-    ;; Floating-point convert to higher precision long, 32-bit elements
-    (Fcvtl32)
-    ;; Shift left long (by element size), 8-bit elements
-    (Shll8)
-    ;; Shift left long (by element size), 16-bit elements
-    (Shll16)
-    ;; Shift left long (by element size), 32-bit elements
-    (Shll32)
-))
-
-;; A vector narrowing operation with one argument.
-(type VecRRNarrowOp
-  (enum
-    ;; Extract narrow.
-    (Xtn)
-    ;; Signed saturating extract narrow.
-    (Sqxtn)
-    ;; Signed saturating extract unsigned narrow.
-    (Sqxtun)
-    ;; Unsigned saturating extract narrow.
-    (Uqxtn)
-    ;; Floating-point convert to lower precision narrow.
-    (Fcvtn)
-))
-
-(type VecRRRLongOp
-  (enum
-    ;; Signed multiply long.
-    (Smull8)
-    (Smull16)
-    (Smull32)
-    ;; Unsigned multiply long.
-    (Umull8)
-    (Umull16)
-    (Umull32)
-))
-
-(type VecRRRLongModOp
-  (enum
-    ;; Unsigned multiply add long
-    (Umlal8)
-    (Umlal16)
-    (Umlal32)
-))
-
-;; A vector operation on a pair of elements with one register.
-(type VecPairOp
-  (enum
-    ;; Add pair of elements
-    (Addp)
-))
-
-;; 1-operand vector instruction that extends elements of the input register
-;; and operates on a pair of elements.
-(type VecRRPairLongOp
-  (enum
-    ;; Sign extend and add pair of elements
-    (Saddlp8)
-    (Saddlp16)
-    ;; Unsigned extend and add pair of elements
-    (Uaddlp8)
-    (Uaddlp16)
-))
-
-;; An operation across the lanes of vectors.
-(type VecLanesOp
-  (enum
-    ;; Integer addition across a vector
-    (Addv)
-    ;; Unsigned minimum across a vector
-    (Uminv)
-))
-
-;; A shift-by-immediate operation on each lane of a vector.
-(type VecShiftImmOp
-  (enum
-    ;; Unsigned shift left
-    (Shl)
-    ;; Unsigned shift right
-    (Ushr)
-    ;; Signed shift right
-    (Sshr)
-))
-
-;; Destructive shift-by-immediate operation on each lane of a vector.
-(type VecShiftImmModOp
-  (enum
-    ;; Shift left and insert
-    (Sli)
-))
-
-;; Atomic read-modify-write operations with acquire-release semantics
-(type AtomicRMWOp
-  (enum
-    (Add)
-    (Clr)
-    (Eor)
-    (Set)
-    (Smax)
-    (Smin)
-    (Umax)
-    (Umin)
-    (Swp)
-))
-
-;; Atomic read-modify-write operations, with acquire-release semantics,
-;; implemented with a loop.
-(type AtomicRMWLoopOp
-  (enum
-    (Add)
-    (Sub)
-    (And)
-    (Nand)
-    (Eor)
-    (Orr)
-    (Smax)
-    (Smin)
-    (Umax)
-    (Umin)
-    (Xchg)
-))
-
-;; Keys for instruction address PACs
-(type APIKey
-  (enum
-    (A)
-    (B)
-))
-
-;; Branch target types
-(type BranchTargetType
-  (enum
-    (None)
-    (C)
-    (J)
-    (JC)
-))
-
-;; Extractors for target features ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(decl pure sign_return_address_disabled () Unit)
-(extern constructor sign_return_address_disabled sign_return_address_disabled)
-
-(decl use_lse () Inst)
-(extern extractor use_lse use_lse)
-
-;; Extractor helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl pure imm_logic_from_u64 (Type u64) ImmLogic)
-(extern constructor imm_logic_from_u64 imm_logic_from_u64)
-
-(decl pure imm_logic_from_imm64 (Type Imm64) ImmLogic)
-(extern constructor imm_logic_from_imm64 imm_logic_from_imm64)
-
-(decl pure imm_shift_from_imm64 (Type Imm64) ImmShift)
-(extern constructor imm_shift_from_imm64 imm_shift_from_imm64)
-
-(decl imm_shift_from_u8 (u8) ImmShift)
-(extern constructor imm_shift_from_u8 imm_shift_from_u8)
-
-(decl imm12_from_u64 (Imm12) u64)
-(extern extractor imm12_from_u64 imm12_from_u64)
-
-(decl u8_into_uimm5 (u8) UImm5)
-(extern constructor u8_into_uimm5 u8_into_uimm5)
-
-(decl u8_into_imm12 (u8) Imm12)
-(extern constructor u8_into_imm12 u8_into_imm12)
-
-(decl u64_into_imm_logic (Type u64) ImmLogic)
-(extern constructor u64_into_imm_logic u64_into_imm_logic)
-
-(decl branch_target (VecMachLabel u8) BranchTarget)
-(extern constructor branch_target branch_target)
-
-(decl targets_jt_size (VecMachLabel) u32)
-(extern constructor targets_jt_size targets_jt_size)
-
-(decl targets_jt_space (VecMachLabel) CodeOffset)
-(extern constructor targets_jt_space targets_jt_space)
-
-(decl targets_jt_info (VecMachLabel) BoxJTSequenceInfo)
-(extern constructor targets_jt_info targets_jt_info)
-
-;; Calculate the minimum floating-point bound for a conversion to floating
-;; point from an integer type.
-;; Accepts whether the output is signed, the size of the input
-;; floating point type in bits, and the size of the output integer type
-;; in bits.
-(decl min_fp_value (bool u8 u8) Reg)
-(extern constructor min_fp_value min_fp_value)
-
-;; Calculate the maximum floating-point bound for a conversion to floating
-;; point from an integer type.
-;; Accepts whether the output is signed, the size of the input
-;; floating point type in bits, and the size of the output integer type
-;; in bits.
-(decl max_fp_value (bool u8 u8) Reg)
-(extern constructor max_fp_value max_fp_value)
-
-;; Constructs an FPUOpRI.Ushr* given the size in bits of the value (or lane)
-;; and the amount to shift by.
-(decl fpu_op_ri_ushr (u8 u8) FPUOpRI)
-(extern constructor fpu_op_ri_ushr fpu_op_ri_ushr)
-
-;; Constructs an FPUOpRIMod.Sli* given the size in bits of the value (or lane)
-;; and the amount to shift by.
-(decl fpu_op_ri_sli (u8 u8) FPUOpRIMod)
-(extern constructor fpu_op_ri_sli fpu_op_ri_sli)
-
-(decl imm12_from_negated_u64 (Imm12) u64)
-(extern extractor imm12_from_negated_u64 imm12_from_negated_u64)
-
-(decl pure lshr_from_u64 (Type u64) ShiftOpAndAmt)
-(extern constructor lshr_from_u64 lshr_from_u64)
-
-(decl pure lshl_from_imm64 (Type Imm64) ShiftOpAndAmt)
-(extern constructor lshl_from_imm64 lshl_from_imm64)
-
-(decl pure lshl_from_u64 (Type u64) ShiftOpAndAmt)
-(extern constructor lshl_from_u64 lshl_from_u64)
-
-(decl integral_ty (Type) Type)
-(extern extractor integral_ty integral_ty)
-
-(decl valid_atomic_transaction (Type) Type)
-(extern extractor valid_atomic_transaction valid_atomic_transaction)
-
-(decl pure is_zero_simm9 (SImm9) Unit)
-(extern constructor is_zero_simm9 is_zero_simm9)
-
-(decl pure is_zero_uimm12 (UImm12Scaled) Unit)
-(extern constructor is_zero_uimm12 is_zero_uimm12)
-
-;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
-(decl imm12_from_value (Imm12) Value)
-(extractor
-  (imm12_from_value n)
-  (iconst (u64_from_imm64 (imm12_from_u64 n))))
-
-;; Same as `imm12_from_value`, but tries negating the constant value.
-(decl imm12_from_negated_value (Imm12) Value)
-(extractor
-  (imm12_from_negated_value n)
-  (iconst (u64_from_imm64 (imm12_from_negated_u64 n))))
-
-;; Helper type to represent a value and an extend operation fused together.
-(type ExtendedValue extern (enum))
-(decl extended_value_from_value (ExtendedValue) Value)
-(extern extractor extended_value_from_value extended_value_from_value)
-
-;; Constructors used to poke at the fields of an `ExtendedValue`.
-(decl put_extended_in_reg (ExtendedValue) Reg)
-(extern constructor put_extended_in_reg put_extended_in_reg)
-(decl get_extended_op (ExtendedValue) ExtendOp)
-(extern constructor get_extended_op get_extended_op)
-
-(decl nzcv (bool bool bool bool) NZCV)
-(extern constructor nzcv nzcv)
-
-(decl cond_br_zero (Reg) CondBrKind)
-(extern constructor cond_br_zero cond_br_zero)
-
-(decl cond_br_not_zero (Reg) CondBrKind)
-(extern constructor cond_br_not_zero cond_br_not_zero)
-
-(decl cond_br_cond (Cond) CondBrKind)
-(extern constructor cond_br_cond cond_br_cond)
-
-(decl pair_amode (Value u32) PairAMode)
-(extern constructor pair_amode pair_amode)
-
-;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Helper for creating the zero register.
-(decl zero_reg () Reg)
-(extern constructor zero_reg zero_reg)
-
-(decl fp_reg () Reg)
-(extern constructor fp_reg fp_reg)
-
-(decl stack_reg () Reg)
-(extern constructor stack_reg stack_reg)
-
-(decl writable_link_reg () WritableReg)
-(extern constructor writable_link_reg writable_link_reg)
-
-(decl writable_zero_reg () WritableReg)
-(extern constructor writable_zero_reg writable_zero_reg)
-
-(decl value_regs_zero () ValueRegs)
-(rule (value_regs_zero)
-      (value_regs
-            (imm $I64 (ImmExtend.Zero) 0)
-            (imm $I64 (ImmExtend.Zero) 0)))
-
-
-;; Helper for emitting `MInst.Mov` instructions.
-(decl mov (Reg Type) Reg)
-(rule (mov src ty)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.Mov (operand_size ty) dst src))))
-        dst))
-
-;; Helper for emitting `MInst.MovZ` instructions.
-(decl movz (MoveWideConst OperandSize) Reg)
-(rule (movz imm size)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.MovWide (MoveWideOp.MovZ) dst imm size))))
-        dst))
-
-;; Helper for emitting `MInst.MovN` instructions.
-(decl movn (MoveWideConst OperandSize) Reg)
-(rule (movn imm size)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.MovWide (MoveWideOp.MovN) dst imm size))))
-        dst))
-
-;; Helper for emitting `MInst.AluRRImmLogic` instructions.
-(decl alu_rr_imm_logic (ALUOp Type Reg ImmLogic) Reg)
-(rule (alu_rr_imm_logic op ty src imm)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRImmLogic op (operand_size ty) dst src imm))))
-        dst))
-
-;; Helper for emitting `MInst.AluRRImmShift` instructions.
-(decl alu_rr_imm_shift (ALUOp Type Reg ImmShift) Reg)
-(rule (alu_rr_imm_shift op ty src imm)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRImmShift op (operand_size ty) dst src imm))))
-        dst))
-
-;; Helper for emitting `MInst.AluRRR` instructions.
-(decl alu_rrr (ALUOp Type Reg Reg) Reg)
-(rule (alu_rrr op ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRR op (operand_size ty) dst src1 src2))))
-        dst))
-
-;; Helper for emitting `MInst.VecRRR` instructions.
-(decl vec_rrr (VecALUOp Reg Reg VectorSize) Reg)
-(rule (vec_rrr op src1 src2 size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRR op dst src1 src2 size))))
-        dst))
-
-;; Helper for emitting `MInst.FpuRR` instructions.
-(decl fpu_rr (FPUOp1 Reg ScalarSize) Reg)
-(rule (fpu_rr op src size)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuRR op size dst src))))
-       dst))
-
-;; Helper for emitting `MInst.VecRRR` instructions which use three registers,
-;; one of which is both source and output.
-(decl vec_rrr_mod (VecALUModOp Reg Reg Reg VectorSize) Reg)
-(rule (vec_rrr_mod op src1 src2 src3 size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_1 Unit (emit (MInst.VecRRRMod op dst src1 src2 src3 size))))
-        dst))
-
-(decl fpu_rri (FPUOpRI Reg) Reg)
-(rule (fpu_rri op src)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuRRI op dst src))))
-        dst))
-
-(decl fpu_rri_mod (FPUOpRIMod Reg Reg) Reg)
-(rule (fpu_rri_mod op dst_src src)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuRRIMod op dst dst_src src))))
-        dst))
-
-;; Helper for emitting `MInst.FpuRRR` instructions.
-(decl fpu_rrr (FPUOp2 Reg Reg ScalarSize) Reg)
-(rule (fpu_rrr op src1 src2 size)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuRRR op size dst src1 src2))))
-        dst))
-
-;; Helper for emitting `MInst.FpuRRRR` instructions.
-(decl fpu_rrrr (FPUOp3 ScalarSize Reg Reg Reg) Reg)
-(rule (fpu_rrrr size op src1 src2 src3)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuRRRR size op dst src1 src2 src3))))
-        dst))
-
-;; Helper for emitting `MInst.FpuCmp` instructions.
-(decl fpu_cmp (ScalarSize Reg Reg) ProducesFlags)
-(rule (fpu_cmp size rn rm)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.FpuCmp size rn rm)))
-
-;; Helper for emitting `MInst.VecLanes` instructions.
-(decl vec_lanes (VecLanesOp Reg VectorSize) Reg)
-(rule (vec_lanes op src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecLanes op dst src size))))
-        dst))
-
-;; Helper for emitting `MInst.VecShiftImm` instructions.
-(decl vec_shift_imm (VecShiftImmOp u8 Reg VectorSize) Reg)
-(rule (vec_shift_imm op imm src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecShiftImm op dst src size imm))))
-        dst))
-
-;; Helper for emitting `MInst.VecDup` instructions.
-(decl vec_dup (Reg VectorSize) Reg)
-(rule (vec_dup src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecDup dst src size))))
-        dst))
-
-;; Helper for emitting `MInst.VecDupFromFpu` instructions.
-(decl vec_dup_from_fpu (Reg VectorSize) Reg)
-(rule (vec_dup_from_fpu src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecDupFromFpu dst src size))))
-        dst))
-
-;; Helper for emitting `MInst.AluRRImm12` instructions.
-(decl alu_rr_imm12 (ALUOp Type Reg Imm12) Reg)
-(rule (alu_rr_imm12 op ty src imm)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRImm12 op (operand_size ty) dst src imm))))
-        dst))
-
-;; Helper for emitting `MInst.AluRRRShift` instructions.
-(decl alu_rrr_shift (ALUOp Type Reg Reg ShiftOpAndAmt) Reg)
-(rule (alu_rrr_shift op ty src1 src2 shift)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRRShift op (operand_size ty) dst src1 src2 shift))))
-        dst))
-
-;; Helper for emitting `cmp` instructions, setting flags, with a right-shifted
-;; second operand register.
-(decl cmp_rr_shift (OperandSize Reg Reg u64) ProducesFlags)
-(rule (cmp_rr_shift size src1 src2 shift_amount)
-      (if-let shift (lshr_from_u64 $I64 shift_amount))
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.AluRRRShift (ALUOp.SubS) size (writable_zero_reg)
-        src1 src2 shift)))
-
-;; Helper for emitting `MInst.AluRRRExtend` instructions.
-(decl alu_rrr_extend (ALUOp Type Reg Reg ExtendOp) Reg)
-(rule (alu_rrr_extend op ty src1 src2 extend)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRRExtend op (operand_size ty) dst src1 src2 extend))))
-        dst))
-
-;; Same as `alu_rrr_extend`, but takes an `ExtendedValue` packed "pair" instead
-;; of a `Reg` and an `ExtendOp`.
-(decl alu_rr_extend_reg (ALUOp Type Reg ExtendedValue) Reg)
-(rule (alu_rr_extend_reg op ty src1 extended_reg)
-      (let ((src2 Reg (put_extended_in_reg extended_reg))
-            (extend ExtendOp (get_extended_op extended_reg)))
-        (alu_rrr_extend op ty src1 src2 extend)))
-
-;; Helper for emitting `MInst.AluRRRR` instructions.
-(decl alu_rrrr (ALUOp3 Type Reg Reg Reg) Reg)
-(rule (alu_rrrr op ty src1 src2 src3)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRRR op (operand_size ty) dst src1 src2 src3))))
-        dst))
-
-;; Helper for emitting `MInst.BitRR` instructions.
-(decl bit_rr (BitOp Type Reg) Reg)
-(rule (bit_rr op ty src)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.BitRR op (operand_size ty) dst src))))
-        dst))
-
-;; Helper for emitting `adds` instructions.
-(decl add_with_flags_paired (Type Reg Reg) ProducesFlags)
-(rule (add_with_flags_paired ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
-         (MInst.AluRRR (ALUOp.AddS) (operand_size ty) dst src1 src2)
-         dst)))
-
-;; Helper for emitting `adds` instructions, setting flags in ambient
-;; state. Used only for `iadd_ifcout`.
-(decl add_with_flags (Type Reg Reg) Reg)
-(rule (add_with_flags ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AluRRR (ALUOp.AddS) (operand_size ty) dst src1 src2))))
-        dst))
-
-;; Helper for emitting `adc` instructions.
-(decl adc_paired (Type Reg Reg) ConsumesFlags)
-(rule (adc_paired ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
-         (MInst.AluRRR (ALUOp.Adc) (operand_size ty) dst src1 src2)
-         dst)))
-
-;; Helper for emitting `subs` instructions.
-(decl sub_with_flags_paired (Type Reg Reg) ProducesFlags)
-(rule (sub_with_flags_paired ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
-         (MInst.AluRRR (ALUOp.SubS) (operand_size ty) dst src1 src2)
-         dst)))
-
-;; Helper for materializing a boolean value into a register from
-;; flags.
-(decl materialize_bool_result (Cond) ConsumesFlags)
-(rule (materialize_bool_result cond)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.CSet dst cond)
-         dst)))
-
-(decl cmn_imm (OperandSize Reg Imm12) ProducesFlags)
-(rule (cmn_imm size src1 src2)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.AluRRImm12 (ALUOp.AddS) size (writable_zero_reg)
-        src1 src2)))
-
-(decl cmp (OperandSize Reg Reg) ProducesFlags)
-(rule (cmp size src1 src2)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.AluRRR (ALUOp.SubS) size (writable_zero_reg)
-        src1 src2)))
-
-(decl cmp_imm (OperandSize Reg Imm12) ProducesFlags)
-(rule (cmp_imm size src1 src2)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.AluRRImm12 (ALUOp.SubS) size (writable_zero_reg)
-        src1 src2)))
-
-(decl cmp64_imm (Reg Imm12) ProducesFlags)
-(rule (cmp64_imm src1 src2)
-      (cmp_imm (OperandSize.Size64) src1 src2))
-
-(decl cmp_extend (OperandSize Reg Reg ExtendOp) ProducesFlags)
-(rule (cmp_extend size src1 src2 extend)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.AluRRRExtend (ALUOp.SubS) size (writable_zero_reg)
-        src1 src2 extend)))
-
-;; Helper for emitting `sbc` instructions.
-(decl sbc_paired (Type Reg Reg) ConsumesFlags)
-(rule (sbc_paired ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
-         (MInst.AluRRR (ALUOp.Sbc) (operand_size ty) dst src1 src2)
-         dst)))
-
-;; Helper for emitting `MInst.VecMisc` instructions.
-(decl vec_misc (VecMisc2 Reg VectorSize) Reg)
-(rule (vec_misc op src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecMisc op dst src size))))
-        dst))
-
-;; Helper for emitting `MInst.VecTbl` instructions.
-(decl vec_tbl (Reg Reg) Reg)
-(rule (vec_tbl rn rm)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecTbl dst rn rm))))
-        dst))
-
-(decl vec_tbl_ext (Reg Reg Reg) Reg)
-(rule (vec_tbl_ext ri rn rm)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecTblExt dst ri rn rm))))
-        dst))
-
-;; Helper for emitting `MInst.VecTbl2` instructions.
-(decl vec_tbl2 (Reg Reg Reg Type) Reg)
-(rule (vec_tbl2 rn rn2 rm ty)
-      (let (
-            (dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecTbl2 dst rn rn2 rm)))
-        )
-        dst))
-
-;; Helper for emitting `MInst.VecTbl2Ext` instructions.
-(decl vec_tbl2_ext (Reg Reg Reg Reg Type) Reg)
-(rule (vec_tbl2_ext ri rn rn2 rm ty)
-      (let (
-            (dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecTbl2Ext dst ri rn rn2 rm)))
-        )
-        dst))
-
-;; Helper for emitting `MInst.VecRRRLong` instructions.
-(decl vec_rrr_long (VecRRRLongOp Reg Reg bool) Reg)
-(rule (vec_rrr_long op src1 src2 high_half)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRRLong op dst src1 src2 high_half))))
-        dst))
-
-;; Helper for emitting `MInst.VecRRPairLong` instructions.
-(decl vec_rr_pair_long (VecRRPairLongOp Reg) Reg)
-(rule (vec_rr_pair_long op src)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRPairLong op dst src))))
-        dst))
-
-;; Helper for emitting `MInst.VecRRRLongMod` instructions.
-(decl vec_rrrr_long (VecRRRLongModOp Reg Reg Reg bool) Reg)
-(rule (vec_rrrr_long op src1 src2 src3 high_half)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRRLongMod op dst src1 src2 src3 high_half))))
-        dst))
-
-;; Helper for emitting `MInst.VecRRNarrow` instructions.
-(decl vec_rr_narrow_low (VecRRNarrowOp Reg ScalarSize) Reg)
-(rule (vec_rr_narrow_low op src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRNarrowLow op dst src size))))
-        dst))
-
-;; Helper for emitting `MInst.VecRRNarrow` instructions which update the
-;; high half of the destination register.
-(decl vec_rr_narrow_high (VecRRNarrowOp Reg Reg ScalarSize) Reg)
-(rule (vec_rr_narrow_high op mod src size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRNarrowHigh op dst mod src size))))
-        dst))
-
-;; Helper for emitting `MInst.VecRRLong` instructions.
-(decl vec_rr_long (VecRRLongOp Reg bool) Reg)
-(rule (vec_rr_long op src high_half)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecRRLong op dst src high_half))))
-        dst))
-
-;; Helper for emitting `MInst.FpuCSel32` / `MInst.FpuCSel64`
-;; instructions.
-(decl fpu_csel (Type Cond Reg Reg) ConsumesFlags)
-(rule (fpu_csel $F32 cond if_true if_false)
-      (let ((dst WritableReg (temp_writable_reg $F32)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.FpuCSel32 dst if_true if_false cond)
-         dst)))
-
-(rule (fpu_csel $F64 cond if_true if_false)
-      (let ((dst WritableReg (temp_writable_reg $F64)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.FpuCSel64 dst if_true if_false cond)
-         dst)))
-
-;; Helper for emitting `MInst.VecCSel` instructions.
-(decl vec_csel (Cond Reg Reg) ConsumesFlags)
-(rule (vec_csel cond if_true if_false)
-      (let ((dst WritableReg (temp_writable_reg $I8X16)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.VecCSel dst if_true if_false cond)
-         dst)))
-
-;; Helper for emitting `MInst.FpuRound` instructions.
-(decl fpu_round (FpuRoundMode Reg) Reg)
-(rule (fpu_round op rn)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuRound op dst rn))))
-        dst))
-
-;; Helper for emitting `MInst.FpuMove64` and `MInst.FpuMove128` instructions.
-(decl fpu_move (Type Reg) Reg)
-(rule (fpu_move _ src)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.FpuMove128 dst src))))
-        dst))
-(rule 1 (fpu_move (fits_in_64 _) src)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuMove64 dst src))))
-        dst))
-
-;; Helper for emitting `MInst.MovToFpu` instructions.
-(decl mov_to_fpu (Reg ScalarSize) Reg)
-(rule (mov_to_fpu x size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.MovToFpu dst x size))))
-        dst))
-
-;; Helper for emitting `MInst.MovToVec` instructions.
-(decl mov_to_vec (Reg Reg u8 VectorSize) Reg)
-(rule (mov_to_vec src1 src2 lane size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.MovToVec dst src1 src2 lane size))))
-        dst))
-
-;; Helper for emitting `MInst.VecMovElement` instructions.
-(decl mov_vec_elem (Reg Reg u8 u8 VectorSize) Reg)
-(rule (mov_vec_elem src1 src2 dst_idx src_idx size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecMovElement dst src1 src2 dst_idx src_idx size))))
-        dst))
-
-;; Helper for emitting `MInst.MovFromVec` instructions.
-(decl mov_from_vec (Reg u8 ScalarSize) Reg)
-(rule (mov_from_vec rn idx size)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.MovFromVec dst rn idx size))))
-        dst))
-
-;; Helper for emitting `MInst.MovFromVecSigned` instructions.
-(decl mov_from_vec_signed (Reg u8 VectorSize OperandSize) Reg)
-(rule (mov_from_vec_signed rn idx size scalar_size)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.MovFromVecSigned dst rn idx size scalar_size))))
-        dst))
-
-(decl fpu_move_from_vec (Reg u8 VectorSize) Reg)
-(rule (fpu_move_from_vec rn idx size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.FpuMoveFromVec dst rn idx size))))
-        dst))
-
-;; Helper for emitting `MInst.Extend` instructions.
-(decl extend (Reg bool u8 u8) Reg)
-(rule (extend rn signed from_bits to_bits)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.Extend dst rn signed from_bits to_bits))))
-        dst))
-
-;; Helper for emitting `MInst.FpuExtend` instructions.
-(decl fpu_extend (Reg ScalarSize) Reg)
-(rule (fpu_extend src size)
-      (let ((dst WritableReg (temp_writable_reg $F32X4))
-            (_ Unit (emit (MInst.FpuExtend dst src size))))
-        dst))
-
-;; Helper for emitting `MInst.VecExtend` instructions.
-(decl vec_extend (VecExtendOp Reg bool ScalarSize) Reg)
-(rule (vec_extend op src high_half size)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecExtend op dst src high_half size))))
-        dst))
-
-;; Helper for emitting `MInst.VecExtract` instructions.
-(decl vec_extract (Reg Reg u8) Reg)
-(rule (vec_extract src1 src2 idx)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecExtract dst src1 src2 idx))))
-        dst))
-
-;; Helper for emitting `MInst.LoadAcquire` instructions.
-(decl load_acquire (Type MemFlags Reg) Reg)
-(rule (load_acquire ty flags addr)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.LoadAcquire ty dst addr flags))))
-        dst))
-
-;; Helper for emitting `MInst.StoreRelease` instructions.
-(decl store_release (Type MemFlags Reg Reg) SideEffectNoResult)
-(rule (store_release ty flags src addr)
-      (SideEffectNoResult.Inst (MInst.StoreRelease ty src addr flags)))
-
-;; Helper for generating a `tst` instruction.
-;;
-;; Produces a `ProducesFlags` rather than a register or emitted instruction
-;; which must be paired with `with_flags*` helpers.
-(decl tst_imm (Type Reg ImmLogic) ProducesFlags)
-(rule (tst_imm ty reg imm)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.AluRRImmLogic (ALUOp.AndS)
-                            (operand_size ty)
-                            (writable_zero_reg)
-                            reg
-                            imm)))
-
-;; Helper for generating a `CSel` instruction.
-;;
-;; Note that this doesn't actually emit anything, instead it produces a
-;; `ConsumesFlags` instruction which must be consumed with `with_flags*`
-;; helpers.
-(decl csel (Cond Reg Reg) ConsumesFlags)
-(rule (csel cond if_true if_false)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.CSel dst cond if_true if_false)
-         dst)))
-
-;; Helper for constructing `cset` instructions.
-(decl cset (Cond) ConsumesFlags)
-(rule (cset cond)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg (MInst.CSet dst cond) dst)))
-
-;; Helper for constructing `cset` instructions, when the flags producer will
-;; also return a value.
-(decl cset_paired (Cond) ConsumesFlags)
-(rule (cset_paired cond)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer (MInst.CSet dst cond) dst)))
-
-;; Helper for constructing `csetm` instructions.
-(decl csetm (Cond) ConsumesFlags)
-(rule (csetm cond)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg (MInst.CSetm dst cond) dst)))
-
-;; Helper for generating a `CSNeg` instruction.
-;;
-;; Note that this doesn't actually emit anything, instead it produces a
-;; `ConsumesFlags` instruction which must be consumed with `with_flags*`
-;; helpers.
-(decl csneg (Cond Reg Reg) ConsumesFlags)
-(rule (csneg cond if_true if_false)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.CSNeg dst cond if_true if_false)
-         dst)))
-
-;; Helper for generating `MInst.CCmp` instructions.
-;; Creates a new `ProducesFlags` from the supplied `ProducesFlags` followed
-;; immediately by the `MInst.CCmp` instruction.
-(decl ccmp (OperandSize Reg Reg NZCV Cond ProducesFlags) ProducesFlags)
-(rule (ccmp size rn rm nzcv cond inst_input)
-      (produces_flags_append inst_input (MInst.CCmp size rn rm nzcv cond)))
-
-;; Helper for generating `MInst.CCmpImm` instructions.
-(decl ccmp_imm (OperandSize Reg UImm5 NZCV Cond) ConsumesFlags)
-(rule 1 (ccmp_imm size rn imm nzcv cond)
-      (let ((dst WritableReg (temp_writable_reg $I64)))
-        (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
-         (MInst.CCmpImm size rn imm nzcv cond)
-         (MInst.CSet dst cond)
-         (value_reg dst))))
-
-;; Helpers for generating `add` instructions.
-
-(decl add (Type Reg Reg) Reg)
-(rule (add ty x y) (alu_rrr (ALUOp.Add) ty x y))
-
-(decl add_imm (Type Reg Imm12) Reg)
-(rule (add_imm ty x y) (alu_rr_imm12 (ALUOp.Add) ty x y))
-
-(decl add_extend (Type Reg ExtendedValue) Reg)
-(rule (add_extend ty x y) (alu_rr_extend_reg (ALUOp.Add) ty x y))
-
-(decl add_extend_op (Type Reg Reg ExtendOp) Reg)
-(rule (add_extend_op ty x y extend) (alu_rrr_extend (ALUOp.Add) ty x y extend))
-
-(decl add_shift (Type Reg Reg ShiftOpAndAmt) Reg)
-(rule (add_shift ty x y z) (alu_rrr_shift (ALUOp.Add) ty x y z))
-
-(decl add_vec (Reg Reg VectorSize) Reg)
-(rule (add_vec x y size) (vec_rrr (VecALUOp.Add) x y size))
-
-;; Helpers for generating `sub` instructions.
-
-(decl sub (Type Reg Reg) Reg)
-(rule (sub ty x y) (alu_rrr (ALUOp.Sub) ty x y))
-
-(decl sub_imm (Type Reg Imm12) Reg)
-(rule (sub_imm ty x y) (alu_rr_imm12 (ALUOp.Sub) ty x y))
-
-(decl sub_extend (Type Reg ExtendedValue) Reg)
-(rule (sub_extend ty x y) (alu_rr_extend_reg (ALUOp.Sub) ty x y))
-
-(decl sub_shift (Type Reg Reg ShiftOpAndAmt) Reg)
-(rule (sub_shift ty x y z) (alu_rrr_shift (ALUOp.Sub) ty x y z))
-
-(decl sub_vec (Reg Reg VectorSize) Reg)
-(rule (sub_vec x y size) (vec_rrr (VecALUOp.Sub) x y size))
-
-(decl sub_i128 (ValueRegs ValueRegs) ValueRegs)
-(rule (sub_i128 x y)
-      (let
-          ;; Get the high/low registers for `x`.
-          ((x_regs ValueRegs x)
-           (x_lo Reg (value_regs_get x_regs 0))
-           (x_hi Reg (value_regs_get x_regs 1))
-
-           ;; Get the high/low registers for `y`.
-           (y_regs ValueRegs y)
-           (y_lo Reg (value_regs_get y_regs 0))
-           (y_hi Reg (value_regs_get y_regs 1)))
-        ;; the actual subtraction is `subs` followed by `sbc` which comprises
-        ;; the low/high bits of the result
-        (with_flags
-          (sub_with_flags_paired $I64 x_lo y_lo)
-          (sbc_paired $I64 x_hi y_hi))))
-
-;; Helpers for generating `madd` instructions.
-
-(decl madd (Type Reg Reg Reg) Reg)
-(rule (madd ty x y z) (alu_rrrr (ALUOp3.MAdd) ty x y z))
-
-;; Helpers for generating `msub` instructions.
-
-(decl msub (Type Reg Reg Reg) Reg)
-(rule (msub ty x y z) (alu_rrrr (ALUOp3.MSub) ty x y z))
-
-;; Helper for generating `uqadd` instructions.
-(decl uqadd (Reg Reg VectorSize) Reg)
-(rule (uqadd x y size) (vec_rrr (VecALUOp.Uqadd) x y size))
-
-;; Helper for generating `sqadd` instructions.
-(decl sqadd (Reg Reg VectorSize) Reg)
-(rule (sqadd x y size) (vec_rrr (VecALUOp.Sqadd) x y size))
-
-;; Helper for generating `uqsub` instructions.
-(decl uqsub (Reg Reg VectorSize) Reg)
-(rule (uqsub x y size) (vec_rrr (VecALUOp.Uqsub) x y size))
-
-;; Helper for generating `sqsub` instructions.
-(decl sqsub (Reg Reg VectorSize) Reg)
-(rule (sqsub x y size) (vec_rrr (VecALUOp.Sqsub) x y size))
-
-;; Helper for generating `umulh` instructions.
-(decl umulh (Type Reg Reg) Reg)
-(rule (umulh ty x y) (alu_rrr (ALUOp.UMulH) ty x y))
-
-;; Helper for generating `smulh` instructions.
-(decl smulh (Type Reg Reg) Reg)
-(rule (smulh ty x y) (alu_rrr (ALUOp.SMulH) ty x y))
-
-;; Helper for generating `mul` instructions.
-(decl mul (Reg Reg VectorSize) Reg)
-(rule (mul x y size) (vec_rrr (VecALUOp.Mul) x y size))
-
-;; Helper for generating `neg` instructions.
-(decl neg (Reg VectorSize) Reg)
-(rule (neg x size) (vec_misc (VecMisc2.Neg) x size))
-
-;; Helper for generating `rev64` instructions.
-(decl rev64 (Reg VectorSize) Reg)
-(rule (rev64 x size) (vec_misc (VecMisc2.Rev64) x size))
-
-;; Helper for generating `xtn` instructions.
-(decl xtn (Reg ScalarSize) Reg)
-(rule (xtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Xtn) x size))
-
-;; Helper for generating `fcvtn` instructions.
-(decl fcvtn (Reg ScalarSize) Reg)
-(rule (fcvtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Fcvtn) x size))
-
-;; Helper for generating `sqxtn` instructions.
-(decl sqxtn (Reg ScalarSize) Reg)
-(rule (sqxtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Sqxtn) x size))
-
-;; Helper for generating `sqxtn2` instructions.
-(decl sqxtn2 (Reg Reg ScalarSize) Reg)
-(rule (sqxtn2 x y size) (vec_rr_narrow_high (VecRRNarrowOp.Sqxtn) x y size))
-
-;; Helper for generating `sqxtun` instructions.
-(decl sqxtun (Reg ScalarSize) Reg)
-(rule (sqxtun x size) (vec_rr_narrow_low (VecRRNarrowOp.Sqxtun) x size))
-
-;; Helper for generating `sqxtun2` instructions.
-(decl sqxtun2 (Reg Reg ScalarSize) Reg)
-(rule (sqxtun2 x y size) (vec_rr_narrow_high (VecRRNarrowOp.Sqxtun) x y size))
-
-;; Helper for generating `uqxtn` instructions.
-(decl uqxtn (Reg ScalarSize) Reg)
-(rule (uqxtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Uqxtn) x size))
-
-;; Helper for generating `uqxtn2` instructions.
-(decl uqxtn2 (Reg Reg ScalarSize) Reg)
-(rule (uqxtn2 x y size) (vec_rr_narrow_high (VecRRNarrowOp.Uqxtn) x y size))
-
-;; Helper for generating `fence` instructions.
-(decl aarch64_fence () SideEffectNoResult)
-(rule (aarch64_fence)
-      (SideEffectNoResult.Inst (MInst.Fence)))
-
-;; Helper for generating `csdb` instructions.
-(decl csdb () SideEffectNoResult)
-(rule (csdb)
-      (SideEffectNoResult.Inst (MInst.Csdb)))
-
-;; Helper for generating `brk` instructions.
-(decl brk () SideEffectNoResult)
-(rule (brk)
-      (SideEffectNoResult.Inst (MInst.Brk)))
-
-;; Helper for generating `addp` instructions.
-(decl addp (Reg Reg VectorSize) Reg)
-(rule (addp x y size) (vec_rrr (VecALUOp.Addp) x y size))
-
-;; Helper for generating `zip1` instructions.
-(decl zip1 (Reg Reg VectorSize) Reg)
-(rule (zip1 x y size) (vec_rrr (VecALUOp.Zip1) x y size))
-
-;; Helper for generating vector `abs` instructions.
-(decl vec_abs (Reg VectorSize) Reg)
-(rule (vec_abs x size) (vec_misc (VecMisc2.Abs) x size))
-
-;; Helper for generating instruction sequences to calculate a scalar absolute
-;; value.
-(decl abs (OperandSize Reg) Reg)
-(rule (abs size x)
-      (value_regs_get (with_flags (cmp_imm size x (u8_into_imm12 0))
-                                  (csneg (Cond.Gt) x x)) 0))
-
-;; Helper for generating `addv` instructions.
-(decl addv (Reg VectorSize) Reg)
-(rule (addv x size) (vec_lanes (VecLanesOp.Addv) x size))
-
-;; Helper for generating `shll32` instructions.
-(decl shll32 (Reg bool) Reg)
-(rule (shll32 x high_half) (vec_rr_long (VecRRLongOp.Shll32) x high_half))
-
-;; Helpers for generating `addlp` instructions.
-
-(decl saddlp8 (Reg) Reg)
-(rule (saddlp8 x) (vec_rr_pair_long (VecRRPairLongOp.Saddlp8) x))
-
-(decl saddlp16 (Reg) Reg)
-(rule (saddlp16 x) (vec_rr_pair_long (VecRRPairLongOp.Saddlp16) x))
-
-(decl uaddlp8 (Reg) Reg)
-(rule (uaddlp8 x) (vec_rr_pair_long (VecRRPairLongOp.Uaddlp8) x))
-
-(decl uaddlp16 (Reg) Reg)
-(rule (uaddlp16 x) (vec_rr_pair_long (VecRRPairLongOp.Uaddlp16) x))
-
-;; Helper for generating `umlal32` instructions.
-(decl umlal32 (Reg Reg Reg bool) Reg)
-(rule (umlal32 x y z high_half) (vec_rrrr_long (VecRRRLongModOp.Umlal32) x y z high_half))
-
-;; Helper for generating `smull8` instructions.
-(decl smull8 (Reg Reg bool) Reg)
-(rule (smull8 x y high_half) (vec_rrr_long (VecRRRLongOp.Smull8) x y high_half))
-
-;; Helper for generating `umull8` instructions.
-(decl umull8 (Reg Reg bool) Reg)
-(rule (umull8 x y high_half) (vec_rrr_long (VecRRRLongOp.Umull8) x y high_half))
-
-;; Helper for generating `smull16` instructions.
-(decl smull16 (Reg Reg bool) Reg)
-(rule (smull16 x y high_half) (vec_rrr_long (VecRRRLongOp.Smull16) x y high_half))
-
-;; Helper for generating `umull16` instructions.
-(decl umull16 (Reg Reg bool) Reg)
-(rule (umull16 x y high_half) (vec_rrr_long (VecRRRLongOp.Umull16) x y high_half))
-
-;; Helper for generating `smull32` instructions.
-(decl smull32 (Reg Reg bool) Reg)
-(rule (smull32 x y high_half) (vec_rrr_long (VecRRRLongOp.Smull32) x y high_half))
-
-;; Helper for generating `umull32` instructions.
-(decl umull32 (Reg Reg bool) Reg)
-(rule (umull32 x y high_half) (vec_rrr_long (VecRRRLongOp.Umull32) x y high_half))
-
-;; Helper for generating `asr` instructions.
-(decl asr (Type Reg Reg) Reg)
-(rule (asr ty x y) (alu_rrr (ALUOp.Asr) ty x y))
-
-(decl asr_imm (Type Reg ImmShift) Reg)
-(rule (asr_imm ty x imm) (alu_rr_imm_shift (ALUOp.Asr) ty x imm))
-
-;; Helper for generating `lsr` instructions.
-(decl lsr (Type Reg Reg) Reg)
-(rule (lsr ty x y) (alu_rrr (ALUOp.Lsr) ty x y))
-
-(decl lsr_imm (Type Reg ImmShift) Reg)
-(rule (lsr_imm ty x imm) (alu_rr_imm_shift (ALUOp.Lsr) ty x imm))
-
-;; Helper for generating `lsl` instructions.
-(decl lsl (Type Reg Reg) Reg)
-(rule (lsl ty x y) (alu_rrr (ALUOp.Lsl) ty x y))
-
-(decl lsl_imm (Type Reg ImmShift) Reg)
-(rule (lsl_imm ty x imm) (alu_rr_imm_shift (ALUOp.Lsl) ty x imm))
-
-;; Helper for generating `udiv` instructions.
-(decl a64_udiv (Type Reg Reg) Reg)
-(rule (a64_udiv ty x y) (alu_rrr (ALUOp.UDiv) ty x y))
-
-;; Helper for generating `sdiv` instructions.
-(decl a64_sdiv (Type Reg Reg) Reg)
-(rule (a64_sdiv ty x y) (alu_rrr (ALUOp.SDiv) ty x y))
-
-;; Helper for generating `not` instructions.
-(decl not (Reg VectorSize) Reg)
-(rule (not x size) (vec_misc (VecMisc2.Not) x size))
-
-;; Helpers for generating `orr_not` instructions.
-
-(decl orr_not (Type Reg Reg) Reg)
-(rule (orr_not ty x y) (alu_rrr (ALUOp.OrrNot) ty x y))
-
-(decl orr_not_shift (Type Reg Reg ShiftOpAndAmt) Reg)
-(rule (orr_not_shift ty x y shift) (alu_rrr_shift (ALUOp.OrrNot) ty x y shift))
-
-;; Helpers for generating `orr` instructions.
-
-(decl orr (Type Reg Reg) Reg)
-(rule (orr ty x y) (alu_rrr (ALUOp.Orr) ty x y))
-
-(decl orr_imm (Type Reg ImmLogic) Reg)
-(rule (orr_imm ty x y) (alu_rr_imm_logic (ALUOp.Orr) ty x y))
-
-(decl orr_vec (Reg Reg VectorSize) Reg)
-(rule (orr_vec x y size) (vec_rrr (VecALUOp.Orr) x y size))
-
-;; Helpers for generating `and` instructions.
-
-(decl and_reg (Type Reg Reg) Reg)
-(rule (and_reg ty x y) (alu_rrr (ALUOp.And) ty x y))
-
-(decl and_imm (Type Reg ImmLogic) Reg)
-(rule (and_imm ty x y) (alu_rr_imm_logic (ALUOp.And) ty x y))
-
-(decl and_vec (Reg Reg VectorSize) Reg)
-(rule (and_vec x y size) (vec_rrr (VecALUOp.And) x y size))
-
-;; Helpers for generating `eor` instructions.
-(decl eor_vec (Reg Reg VectorSize) Reg)
-(rule (eor_vec x y size) (vec_rrr (VecALUOp.Eor) x y size))
-
-;; Helpers for generating `bic` instructions.
-
-(decl bic (Type Reg Reg) Reg)
-(rule (bic ty x y) (alu_rrr (ALUOp.AndNot) ty x y))
-
-(decl bic_vec (Reg Reg VectorSize) Reg)
-(rule (bic_vec x y size) (vec_rrr (VecALUOp.Bic) x y size))
-
-;; Helpers for generating `sshl` instructions.
-(decl sshl (Reg Reg VectorSize) Reg)
-(rule (sshl x y size) (vec_rrr (VecALUOp.Sshl) x y size))
-
-;; Helpers for generating `ushl` instructions.
-(decl ushl (Reg Reg VectorSize) Reg)
-(rule (ushl x y size) (vec_rrr (VecALUOp.Ushl) x y size))
-
-;; Helpers for generating `rotr` instructions.
-
-(decl a64_rotr (Type Reg Reg) Reg)
-(rule (a64_rotr ty x y) (alu_rrr (ALUOp.RotR) ty x y))
-
-(decl a64_rotr_imm (Type Reg ImmShift) Reg)
-(rule (a64_rotr_imm ty x y) (alu_rr_imm_shift (ALUOp.RotR) ty x y))
-
-;; Helpers for generating `rbit` instructions.
-
-(decl rbit (Type Reg) Reg)
-(rule (rbit ty x) (bit_rr (BitOp.RBit) ty x))
-
-;; Helpers for generating `clz` instructions.
-
-(decl a64_clz (Type Reg) Reg)
-(rule (a64_clz ty x) (bit_rr (BitOp.Clz) ty x))
-
-;; Helpers for generating `cls` instructions.
-
-(decl a64_cls (Type Reg) Reg)
-(rule (a64_cls ty x) (bit_rr (BitOp.Cls) ty x))
-
-;; Helpers for generating `rev` instructions
-
-(decl a64_rev16 (Type Reg) Reg)
-(rule (a64_rev16 ty x) (bit_rr (BitOp.Rev16) ty x))
-
-(decl a64_rev32 (Type Reg) Reg)
-(rule (a64_rev32 ty x) (bit_rr (BitOp.Rev32) ty x))
-
-(decl a64_rev64 (Type Reg) Reg)
-(rule (a64_rev64 ty x) (bit_rr (BitOp.Rev64) ty x))
-
-;; Helpers for generating `eon` instructions.
-
-(decl eon (Type Reg Reg) Reg)
-(rule (eon ty x y) (alu_rrr (ALUOp.EorNot) ty x y))
-
-;; Helpers for generating `cnt` instructions.
-
-(decl vec_cnt (Reg VectorSize) Reg)
-(rule (vec_cnt x size) (vec_misc (VecMisc2.Cnt) x size))
-
-;; Helpers for generating a `bsl` instruction.
-
-(decl bsl (Type Reg Reg Reg) Reg)
-(rule (bsl ty c x y)
-      (vec_rrr_mod (VecALUModOp.Bsl) c x y (vector_size ty)))
-
-;; Helper for generating a `udf` instruction.
-
-(decl udf (TrapCode) SideEffectNoResult)
-(rule (udf trap_code)
-      (SideEffectNoResult.Inst (MInst.Udf trap_code)))
-
-;; Helpers for generating various load instructions, with varying
-;; widths and sign/zero-extending properties.
-(decl aarch64_uload8 (AMode MemFlags) Reg)
-(rule (aarch64_uload8 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.ULoad8 dst amode flags))))
-        dst))
-(decl aarch64_sload8 (AMode MemFlags) Reg)
-(rule (aarch64_sload8 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.SLoad8 dst amode flags))))
-        dst))
-(decl aarch64_uload16 (AMode MemFlags) Reg)
-(rule (aarch64_uload16 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.ULoad16 dst amode flags))))
-        dst))
-(decl aarch64_sload16 (AMode MemFlags) Reg)
-(rule (aarch64_sload16 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.SLoad16 dst amode flags))))
-        dst))
-(decl aarch64_uload32 (AMode MemFlags) Reg)
-(rule (aarch64_uload32 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.ULoad32 dst amode flags))))
-        dst))
-(decl aarch64_sload32 (AMode MemFlags) Reg)
-(rule (aarch64_sload32 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.SLoad32 dst amode flags))))
-        dst))
-(decl aarch64_uload64 (AMode MemFlags) Reg)
-(rule (aarch64_uload64 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.ULoad64 dst amode flags))))
-        dst))
-(decl aarch64_fpuload32 (AMode MemFlags) Reg)
-(rule (aarch64_fpuload32 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuLoad32 dst amode flags))))
-        dst))
-(decl aarch64_fpuload64 (AMode MemFlags) Reg)
-(rule (aarch64_fpuload64 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (_ Unit (emit (MInst.FpuLoad64 dst amode flags))))
-        dst))
-(decl aarch64_fpuload128 (AMode MemFlags) Reg)
-(rule (aarch64_fpuload128 amode flags)
-      (let ((dst WritableReg (temp_writable_reg $F64X2))
-            (_ Unit (emit (MInst.FpuLoad128 dst amode flags))))
-        dst))
-(decl aarch64_loadp64 (PairAMode MemFlags) ValueRegs)
-(rule (aarch64_loadp64 amode flags)
-      (let ((dst1 WritableReg (temp_writable_reg $I64))
-            (dst2 WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.LoadP64 dst1 dst2 amode flags))))
-        (value_regs dst1 dst2)))
-
-;; Helpers for generating various store instructions with varying
-;; widths.
-(decl aarch64_store8 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_store8 amode flags val)
-      (SideEffectNoResult.Inst (MInst.Store8 val amode flags)))
-(decl aarch64_store16 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_store16 amode flags val)
-      (SideEffectNoResult.Inst (MInst.Store16 val amode flags)))
-(decl aarch64_store32 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_store32 amode flags val)
-      (SideEffectNoResult.Inst (MInst.Store32 val amode flags)))
-(decl aarch64_store64 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_store64 amode flags val)
-      (SideEffectNoResult.Inst (MInst.Store64 val amode flags)))
-(decl aarch64_fpustore32 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_fpustore32 amode flags val)
-      (SideEffectNoResult.Inst (MInst.FpuStore32 val amode flags)))
-(decl aarch64_fpustore64 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_fpustore64 amode flags val)
-      (SideEffectNoResult.Inst (MInst.FpuStore64 val amode flags)))
-(decl aarch64_fpustore128 (AMode MemFlags Reg) SideEffectNoResult)
-(rule (aarch64_fpustore128 amode flags val)
-      (SideEffectNoResult.Inst (MInst.FpuStore128 val amode flags)))
-(decl aarch64_storep64 (PairAMode MemFlags Reg Reg) SideEffectNoResult)
-(rule (aarch64_storep64 amode flags val1 val2)
-      (SideEffectNoResult.Inst (MInst.StoreP64 val1 val2 amode flags)))
-
-;; Helper for generating a `trapif` instruction.
-
-(decl trap_if (ProducesFlags TrapCode Cond) InstOutput)
-(rule (trap_if flags trap_code cond)
-      (side_effect
-       (with_flags_side_effect flags
-        (ConsumesFlags.ConsumesFlagsSideEffect
-         (MInst.TrapIf (cond_br_cond cond) trap_code)))))
-
-;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Type of extension performed by an immediate helper
-(type ImmExtend
-  (enum
-    (Sign)
-    (Zero)))
-
-;; Arguments:
-;; * Immediate type
-;; * Way to extend the immediate value to the full width of the destination
-;;   register
-;; * Immediate value - only the bits that fit within the type are used and
-;;   extended, while the rest are ignored
-;;
-;; Note that, unlike the convention in the AArch64 backend, this helper leaves
-;; all bits in the destination register in a defined state, i.e. smaller types
-;; such as `I8` are either sign- or zero-extended.
-(decl imm (Type ImmExtend u64) Reg)
-
-;; Weird logical-instruction immediate in ORI using zero register; to simplify,
-;; we only match when we are zero-extending the value.
-(rule 1 (imm (integral_ty ty) (ImmExtend.Zero) k)
-      (if-let n (imm_logic_from_u64 ty k))
-      (orr_imm ty (zero_reg) n))
-
-(decl load_constant64_full (Type ImmExtend u64) Reg)
-(extern constructor load_constant64_full load_constant64_full)
-
-;; Fallback for integral 64-bit constants
-(rule (imm (integral_ty ty) extend n)
-      (load_constant64_full ty extend n))
-
-;; Sign extension helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Place a `Value` into a register, sign extending it to 32-bits
-(decl put_in_reg_sext32 (Value) Reg)
-(rule -1 (put_in_reg_sext32 val @ (value_type (fits_in_32 ty)))
-      (extend val $true (ty_bits ty) 32))
-
-;; 32/64-bit passthrough.
-(rule (put_in_reg_sext32 val @ (value_type $I32)) val)
-(rule (put_in_reg_sext32 val @ (value_type $I64)) val)
-
-;; Place a `Value` into a register, zero extending it to 32-bits
-(decl put_in_reg_zext32 (Value) Reg)
-(rule -1 (put_in_reg_zext32 val @ (value_type (fits_in_32 ty)))
-      (extend val $false (ty_bits ty) 32))
-
-;; 32/64-bit passthrough.
-(rule (put_in_reg_zext32 val @ (value_type $I32)) val)
-(rule (put_in_reg_zext32 val @ (value_type $I64)) val)
-
-;; Place a `Value` into a register, sign extending it to 64-bits
-(decl put_in_reg_sext64 (Value) Reg)
-(rule 1 (put_in_reg_sext64 val @ (value_type (fits_in_32 ty)))
-      (extend val $true (ty_bits ty) 64))
-
-;; 64-bit passthrough.
-(rule (put_in_reg_sext64 val @ (value_type $I64)) val)
-
-;; Place a `Value` into a register, zero extending it to 64-bits
-(decl put_in_reg_zext64 (Value) Reg)
-(rule 1 (put_in_reg_zext64 val @ (value_type (fits_in_32 ty)))
-      (extend val $false (ty_bits ty) 64))
-
-;; 64-bit passthrough.
-(rule (put_in_reg_zext64 val @ (value_type $I64)) val)
-
-;; Misc instruction helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl trap_if_zero_divisor (Reg) Reg)
-(rule (trap_if_zero_divisor reg)
-      (let ((_ Unit (emit (MInst.TrapIf (cond_br_zero reg) (trap_code_division_by_zero)))))
-        reg))
-
-(decl size_from_ty (Type) OperandSize)
-(rule 1 (size_from_ty (fits_in_32 _ty)) (OperandSize.Size32))
-(rule (size_from_ty $I64) (OperandSize.Size64))
-
-;; Check for signed overflow. The only case is min_value / -1.
-;; The following checks must be done in 32-bit or 64-bit, depending
-;; on the input type.
-(decl trap_if_div_overflow (Type Reg Reg) Reg)
-(rule (trap_if_div_overflow ty x y)
-      (let (
-          ;; Check RHS is -1.
-          (_ Unit (emit (MInst.AluRRImm12 (ALUOp.AddS) (operand_size ty) (writable_zero_reg) y (u8_into_imm12 1))))
-
-          ;; Check LHS is min_value, by subtracting 1 and branching if
-          ;; there is overflow.
-          (_ Unit (emit (MInst.CCmpImm (size_from_ty ty)
-                                       x
-                                       (u8_into_uimm5 1)
-                                       (nzcv $false $false $false $false)
-                                       (Cond.Eq))))
-          (_ Unit (emit (MInst.TrapIf (cond_br_cond (Cond.Vs))
-                                      (trap_code_integer_overflow))))
-        )
-        x))
-
-;; Check for unsigned overflow.
-(decl trap_if_overflow (ProducesFlags TrapCode) Reg)
-(rule (trap_if_overflow producer tc)
-      (with_flags_reg
-        producer
-        (ConsumesFlags.ConsumesFlagsSideEffect
-          (MInst.TrapIf (cond_br_cond (Cond.Hs)) tc))))
-
-(decl sink_atomic_load (Inst) Reg)
-(rule (sink_atomic_load x @ (atomic_load _ addr))
-      (let ((_ Unit (sink_inst x)))
-           (put_in_reg addr)))
-
-;; Helper for generating either an `AluRRR`, `AluRRRShift`, or `AluRRImmLogic`
-;; instruction depending on the input. Note that this requires that the `ALUOp`
-;; specified is commutative.
-(decl alu_rs_imm_logic_commutative (ALUOp Type Value Value) Reg)
-
-;; Base case of operating on registers.
-(rule -1 (alu_rs_imm_logic_commutative op ty x y)
-      (alu_rrr op ty x y))
-
-;; Special cases for when one operand is a constant.
-(rule (alu_rs_imm_logic_commutative op ty x (iconst k))
-      (if-let imm (imm_logic_from_imm64 ty k))
-      (alu_rr_imm_logic op ty x imm))
-(rule 1 (alu_rs_imm_logic_commutative op ty (iconst k) x)
-      (if-let imm (imm_logic_from_imm64 ty k))
-      (alu_rr_imm_logic op ty x imm))
-
-;; Special cases for when one operand is shifted left by a constant.
-(rule (alu_rs_imm_logic_commutative op ty x (ishl y (iconst k)))
-      (if-let amt (lshl_from_imm64 ty k))
-      (alu_rrr_shift op ty x y amt))
-(rule 1 (alu_rs_imm_logic_commutative op ty (ishl x (iconst k)) y)
-      (if-let amt (lshl_from_imm64 ty k))
-      (alu_rrr_shift op ty y x amt))
-
-;; Same as `alu_rs_imm_logic_commutative` above, except that it doesn't require
-;; that the operation is commutative.
-(decl alu_rs_imm_logic (ALUOp Type Value Value) Reg)
-(rule -1 (alu_rs_imm_logic op ty x y)
-      (alu_rrr op ty x y))
-(rule (alu_rs_imm_logic op ty x (iconst k))
-      (if-let imm (imm_logic_from_imm64 ty k))
-      (alu_rr_imm_logic op ty x imm))
-(rule (alu_rs_imm_logic op ty x (ishl y (iconst k)))
-      (if-let amt (lshl_from_imm64 ty k))
-      (alu_rrr_shift op ty x y amt))
-
-;; Helper for generating i128 bitops which simply do the same operation to the
-;; hi/lo registers.
-;;
-;; TODO: Support immlogic here
-(decl i128_alu_bitop (ALUOp Type Value Value) ValueRegs)
-(rule (i128_alu_bitop op ty x y)
-      (let (
-          (x_regs ValueRegs (put_in_regs x))
-          (x_lo Reg (value_regs_get x_regs 0))
-          (x_hi Reg (value_regs_get x_regs 1))
-          (y_regs ValueRegs (put_in_regs y))
-          (y_lo Reg (value_regs_get y_regs 0))
-          (y_hi Reg (value_regs_get y_regs 1))
-        )
-        (value_regs
-          (alu_rrr op ty x_lo y_lo)
-          (alu_rrr op ty x_hi y_hi))))
-
-;; Helper for emitting `MInst.VecLoadReplicate` instructions.
-(decl ld1r (Reg VectorSize MemFlags) Reg)
-(rule (ld1r src size flags)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.VecLoadReplicate dst src size flags))))
-        dst))
-
-;; Helper for emitting `MInst.LoadExtName` instructions.
-(decl load_ext_name (BoxExternalName i64) Reg)
-(rule (load_ext_name extname offset)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.LoadExtName dst extname offset))))
-        dst))
-
-;; Helper for emitting `MInst.LoadAddr` instructions.
-(decl load_addr (AMode) Reg)
-
-(rule (load_addr (AMode.UnsignedOffset r imm))
-      (if (is_zero_uimm12 imm))
-      r)
-
-(rule (load_addr (AMode.Unscaled r imm))
-      (if (is_zero_simm9 imm))
-      r)
-
-(rule (load_addr (AMode.RegOffset r 0 _)) r)
-(rule (load_addr (AMode.FPOffset 0 _)) (fp_reg))
-(rule (load_addr (AMode.SPOffset 0 _)) (stack_reg))
-
-(rule -1 (load_addr addr)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.LoadAddr dst addr))))
-        dst))
-
-;; Lower the address of a load or a store.
-(decl amode (Type Value u32) AMode)
-;; TODO: Port lower_address() to ISLE.
-(extern constructor amode amode)
-
-(decl sink_load_into_amode (Type Inst) AMode)
-(rule (sink_load_into_amode ty x @ (load _ addr offset))
-      (let ((_ Unit (sink_inst x)))
-           (amode ty addr offset)))
-
-;; Lower a constant f64.
-(decl constant_f64 (u64) Reg)
-;; TODO: Port lower_constant_f64() to ISLE.
-(extern constructor constant_f64 constant_f64)
-
-;; Lower a constant f128.
-(decl constant_f128 (u128) Reg)
-;; TODO: Port lower_constant_f128() to ISLE.
-(extern constructor constant_f128 constant_f128)
-
-;; Lower a vector splat with a constant parameter.
-(decl splat_const (u64 VectorSize) Reg)
-;; TODO: Port lower_splat_const() to ISLE.
-(extern constructor splat_const splat_const)
-
-;; Lower a FloatCC to a Cond.
-(decl fp_cond_code (FloatCC) Cond)
-;; TODO: Port lower_fp_condcode() to ISLE.
-(extern constructor fp_cond_code fp_cond_code)
-
-;; Lower an integer cond code.
-(decl cond_code (IntCC) Cond)
-;; TODO: Port lower_condcode() to ISLE.
-(extern constructor cond_code cond_code)
-
-;; Invert a condition code.
-(decl invert_cond (Cond) Cond)
-;; TODO: Port cond.invert() to ISLE.
-(extern constructor invert_cond invert_cond)
-
-;; Generate comparison to zero operator from input condition code
-(decl float_cc_cmp_zero_to_vec_misc_op (FloatCC) VecMisc2)
-(extern constructor float_cc_cmp_zero_to_vec_misc_op float_cc_cmp_zero_to_vec_misc_op)
-
-(decl float_cc_cmp_zero_to_vec_misc_op_swap (FloatCC) VecMisc2)
-(extern constructor float_cc_cmp_zero_to_vec_misc_op_swap float_cc_cmp_zero_to_vec_misc_op_swap)
-
-;; Match valid generic compare to zero cases
-(decl fcmp_zero_cond (FloatCC) FloatCC)
-(extern extractor fcmp_zero_cond fcmp_zero_cond)
-
-;; Match not equal compare to zero separately as it requires two output instructions
-(decl fcmp_zero_cond_not_eq (FloatCC) FloatCC)
-(extern extractor fcmp_zero_cond_not_eq fcmp_zero_cond_not_eq)
-
-;; Helper for generating float compare to zero instructions where 2nd argument is zero
-(decl float_cmp_zero (FloatCC Reg VectorSize) Reg)
-(rule (float_cmp_zero cond rn size)
-      (vec_misc (float_cc_cmp_zero_to_vec_misc_op cond) rn size))
-
-;; Helper for generating float compare to zero instructions in case where 1st argument is zero
-(decl float_cmp_zero_swap (FloatCC Reg VectorSize) Reg)
-(rule (float_cmp_zero_swap cond rn size)
-      (vec_misc (float_cc_cmp_zero_to_vec_misc_op_swap cond) rn size))
-
-;; Helper for generating float compare equal to zero instruction
-(decl fcmeq0 (Reg VectorSize) Reg)
-(rule (fcmeq0 rn size)
-      (vec_misc (VecMisc2.Fcmeq0) rn size))
-
-;; Generate comparison to zero operator from input condition code
-(decl int_cc_cmp_zero_to_vec_misc_op (IntCC) VecMisc2)
-(extern constructor int_cc_cmp_zero_to_vec_misc_op int_cc_cmp_zero_to_vec_misc_op)
-
-(decl int_cc_cmp_zero_to_vec_misc_op_swap (IntCC) VecMisc2)
-(extern constructor int_cc_cmp_zero_to_vec_misc_op_swap int_cc_cmp_zero_to_vec_misc_op_swap)
-
-;; Match valid generic compare to zero cases
-(decl icmp_zero_cond (IntCC) IntCC)
-(extern extractor icmp_zero_cond icmp_zero_cond)
-
-;; Match not equal compare to zero separately as it requires two output instructions
-(decl icmp_zero_cond_not_eq (IntCC) IntCC)
-(extern extractor icmp_zero_cond_not_eq icmp_zero_cond_not_eq)
-
-;; Helper for generating int compare to zero instructions where 2nd argument is zero
-(decl int_cmp_zero (IntCC Reg VectorSize) Reg)
-(rule (int_cmp_zero cond rn size)
-      (vec_misc (int_cc_cmp_zero_to_vec_misc_op cond) rn size))
-
-;; Helper for generating int compare to zero instructions in case where 1st argument is zero
-(decl int_cmp_zero_swap (IntCC Reg VectorSize) Reg)
-(rule (int_cmp_zero_swap cond rn size)
-      (vec_misc (int_cc_cmp_zero_to_vec_misc_op_swap cond) rn size))
-
-;; Helper for generating int compare equal to zero instruction
-(decl cmeq0 (Reg VectorSize) Reg)
-(rule (cmeq0 rn size)
-      (vec_misc (VecMisc2.Cmeq0) rn size))
-
-;; Helper for emitting `MInst.AtomicRMW` instructions.
-(decl lse_atomic_rmw (AtomicRMWOp Value Reg Type MemFlags) Reg)
-(rule (lse_atomic_rmw op p r_arg2 ty flags)
-      (let (
-          (r_addr Reg p)
-          (dst WritableReg (temp_writable_reg ty))
-          (_ Unit (emit (MInst.AtomicRMW op r_arg2 dst r_addr ty flags)))
-        )
-        dst))
-
-;; Helper for emitting `MInst.AtomicCAS` instructions.
-(decl lse_atomic_cas (Reg Reg Reg Type MemFlags) Reg)
-(rule (lse_atomic_cas addr expect replace ty flags)
-      (let (
-            (dst WritableReg (temp_writable_reg ty))
-            (_ Unit (emit (MInst.AtomicCAS dst expect replace addr ty flags)))
-          )
-          dst))
-
-;; Helper for emitting `MInst.AtomicRMWLoop` instructions.
-;; - Make sure that both args are in virtual regs, since in effect
-;; we have to do a parallel copy to get them safely to the AtomicRMW input
-;; regs, and that's not guaranteed safe if either is in a real reg.
-;; - Move the args to the preordained AtomicRMW input regs
-;; - And finally, copy the preordained AtomicRMW output reg to its destination.
-(decl atomic_rmw_loop (AtomicRMWLoopOp Reg Reg Type MemFlags) Reg)
-(rule (atomic_rmw_loop op addr operand ty flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (scratch1 WritableReg (temp_writable_reg $I64))
-            (scratch2 WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AtomicRMWLoop ty op flags addr operand dst scratch1 scratch2))))
-        dst))
-
-;; Helper for emitting `MInst.AtomicCASLoop` instructions.
-;; This is very similar to, but not identical to, the AtomicRmw case.  Note
-;; that the AtomicCASLoop sequence does its own masking, so we don't need to worry
-;; about zero-extending narrow (I8/I16/I32) values here.
-;; Make sure that all three args are in virtual regs.  See corresponding comment
-;; for `atomic_rmw_loop` above.
-(decl atomic_cas_loop (Reg Reg Reg Type MemFlags) Reg)
-(rule (atomic_cas_loop addr expect replace ty flags)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (scratch WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.AtomicCASLoop ty flags addr expect replace dst scratch))))
-        dst))
-
-;; Helper for emitting `MInst.MovPReg` instructions.
-(decl mov_from_preg (PReg) Reg)
-(rule (mov_from_preg src)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.MovFromPReg dst src))))
-        dst))
-
-(decl mov_to_preg (PReg Reg) SideEffectNoResult)
-(rule (mov_to_preg dst src)
-      (SideEffectNoResult.Inst (MInst.MovToPReg dst src)))
-
-(decl preg_sp () PReg)
-(extern constructor preg_sp preg_sp)
-
-(decl preg_fp () PReg)
-(extern constructor preg_fp preg_fp)
-
-(decl preg_link () PReg)
-(extern constructor preg_link preg_link)
-
-(decl preg_pinned () PReg)
-(extern constructor preg_pinned preg_pinned)
-
-(decl aarch64_sp () Reg)
-(rule (aarch64_sp)
-      (mov_from_preg (preg_sp)))
-
-(decl aarch64_fp () Reg)
-(rule (aarch64_fp)
-      (mov_from_preg (preg_fp)))
-
-(decl aarch64_link () Reg)
-(rule 1 (aarch64_link)
-      (if (preserve_frame_pointers))
-      (if (sign_return_address_disabled))
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            ;; Even though LR is not an allocatable register, whether it
-            ;; contains the return address for the current function is
-            ;; unknown at this point. For example, this operation may come
-            ;; immediately after a call, in which case LR would not have a
-            ;; valid value. That's why we must obtain the return address from
-            ;; the frame record that corresponds to the current subroutine on
-            ;; the stack; the presence of the record is guaranteed by the
-            ;; `preserve_frame_pointers` setting.
-            (addr AMode (AMode.FPOffset 8 $I64))
-            (_ Unit (emit (MInst.ULoad64 dst addr (mem_flags_trusted)))))
-           dst))
-
-(rule (aarch64_link)
-      (if (preserve_frame_pointers))
-      ;; Similarly to the rule above, we must load the return address from the
-      ;; the frame record. Furthermore, we can use LR as a scratch register
-      ;; because the function will set it to the return address immediately
-      ;; before returning.
-      (let ((addr AMode (AMode.FPOffset 8 $I64))
-            (lr WritableReg (writable_link_reg))
-            (_ Unit (emit (MInst.ULoad64 lr addr (mem_flags_trusted))))
-            (_ Unit (emit (MInst.Xpaclri))))
-           (mov_from_preg (preg_link))))
-
-;; Helper for getting the maximum shift amount for a type.
-
-(decl max_shift (Type) u8)
-(rule (max_shift $F64) 63)
-(rule (max_shift $F32) 31)
-
-;; Helper for generating `fcopysign` instruction sequences.
-
-(decl fcopy_sign (Reg Reg Type) Reg)
-(rule 1 (fcopy_sign x y (ty_scalar_float ty))
-      (let ((dst WritableReg (temp_writable_reg $F64))
-            (tmp Reg (fpu_rri (fpu_op_ri_ushr (ty_bits ty) (max_shift ty)) y))
-            (_ Unit (emit (MInst.FpuRRIMod (fpu_op_ri_sli (ty_bits ty) (max_shift ty)) dst x tmp))))
-       dst))
-(rule (fcopy_sign x y ty @ (multi_lane _ _))
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (tmp Reg (vec_shift_imm (VecShiftImmOp.Ushr) (max_shift (lane_type ty)) y (vector_size ty)))
-            (_ Unit (emit (MInst.VecShiftImmMod (VecShiftImmModOp.Sli) dst x tmp (vector_size ty) (max_shift (lane_type ty))))))
-       dst))
-
-;; Helpers for generating `MInst.FpuToInt` instructions.
-
-(decl fpu_to_int_nan_check (ScalarSize Reg) Reg)
-(rule (fpu_to_int_nan_check size src)
-      (let ((r ValueRegs
-                  (with_flags (fpu_cmp size src src)
-                   (ConsumesFlags.ConsumesFlagsReturnsReg
-                    (MInst.TrapIf (cond_br_cond (Cond.Vs))
-                        (trap_code_bad_conversion_to_integer))
-                    src))))
-       (value_regs_get r 0)))
-
-;; Checks that the value is not less than the minimum bound,
-;; accepting a boolean (whether the type is signed), input type,
-;; output type, and registers containing the source and minimum bound.
-(decl fpu_to_int_underflow_check (bool Type Type Reg Reg) Reg)
-(rule (fpu_to_int_underflow_check $true $F32 (fits_in_16 out_ty) src min)
-      (let ((r ValueRegs
-                  (with_flags (fpu_cmp (ScalarSize.Size32) src min)
-                   (ConsumesFlags.ConsumesFlagsReturnsReg
-                    (MInst.TrapIf (cond_br_cond (Cond.Le))
-                        (trap_code_integer_overflow))
-                    src))))
-       (value_regs_get r 0)))
-(rule (fpu_to_int_underflow_check $true $F64 (fits_in_32 out_ty) src min)
-      (let ((r ValueRegs
-                  (with_flags (fpu_cmp (ScalarSize.Size64) src min)
-                   (ConsumesFlags.ConsumesFlagsReturnsReg
-                    (MInst.TrapIf (cond_br_cond (Cond.Le))
-                        (trap_code_integer_overflow))
-                    src))))
-       (value_regs_get r 0)))
-(rule -1 (fpu_to_int_underflow_check $true in_ty _out_ty src min)
-      (let ((r ValueRegs
-                  (with_flags (fpu_cmp (scalar_size in_ty) src min)
-                   (ConsumesFlags.ConsumesFlagsReturnsReg
-                    (MInst.TrapIf (cond_br_cond (Cond.Lt))
-                        (trap_code_integer_overflow))
-                    src))))
-       (value_regs_get r 0)))
-(rule (fpu_to_int_underflow_check $false in_ty _out_ty src min)
-      (let ((r ValueRegs
-                  (with_flags (fpu_cmp (scalar_size in_ty) src min)
-                   (ConsumesFlags.ConsumesFlagsReturnsReg
-                    (MInst.TrapIf (cond_br_cond (Cond.Le))
-                        (trap_code_integer_overflow))
-                    src))))
-       (value_regs_get r 0)))
-
-(decl fpu_to_int_overflow_check (ScalarSize Reg Reg) Reg)
-(rule (fpu_to_int_overflow_check size src max)
-      (let ((r ValueRegs
-                  (with_flags (fpu_cmp size src max)
-                   (ConsumesFlags.ConsumesFlagsReturnsReg
-                    (MInst.TrapIf (cond_br_cond (Cond.Ge))
-                        (trap_code_integer_overflow))
-                    src))))
-       (value_regs_get r 0)))
-
-;; Emits the appropriate instruction sequence to convert a
-;; floating-point value to an integer, trapping if the value
-;; is a NaN or does not fit in the target type.
-;; Accepts the specific conversion op, the source register,
-;; whether the input is signed, and finally the input and output
-;; types.
-(decl fpu_to_int_cvt (FpuToIntOp Reg bool Type Type) Reg)
-(rule (fpu_to_int_cvt op src signed in_ty out_ty)
-      (let ((size ScalarSize (scalar_size in_ty))
-            (in_bits u8 (ty_bits in_ty))
-            (out_bits u8 (ty_bits out_ty))
-            (src Reg (fpu_to_int_nan_check size src))
-            (min Reg (min_fp_value signed in_bits out_bits))
-            (src Reg (fpu_to_int_underflow_check signed in_ty out_ty src min))
-            (max Reg (max_fp_value signed in_bits out_bits))
-            (src Reg (fpu_to_int_overflow_check size src max)))
-       (fpu_to_int op src)))
-
-;; Emits the appropriate instruction sequence to convert a
-;; floating-point value to an integer, saturating if the value
-;; does not fit in the target type.
-;; Accepts the specific conversion op, the source register,
-;; whether the input is signed, and finally the output type.
-(decl fpu_to_int_cvt_sat (FpuToIntOp Reg bool Type) Reg)
-(rule 1 (fpu_to_int_cvt_sat op src _ $I64)
-      (fpu_to_int op src))
-(rule 1 (fpu_to_int_cvt_sat op src _ $I32)
-      (fpu_to_int op src))
-(rule (fpu_to_int_cvt_sat op src $false (fits_in_16 out_ty))
-      (let ((result Reg (fpu_to_int op src))
-            (max Reg (imm out_ty (ImmExtend.Zero) -1)))
-       (with_flags_reg
-        (cmp (OperandSize.Size32) result max)
-        (csel (Cond.Hi) max result))))
-(rule (fpu_to_int_cvt_sat op src $true (fits_in_16 out_ty))
-      (let ((result Reg (fpu_to_int op src))
-            (max Reg (imm $I32 (ImmExtend.Sign) (signed_max out_ty)))
-            (min Reg (imm $I32 (ImmExtend.Sign) (signed_min out_ty)))
-            (result Reg (with_flags_reg
-                         (cmp (operand_size out_ty) result max)
-                         (csel (Cond.Gt) max result)))
-            (result Reg (with_flags_reg
-                         (cmp (operand_size out_ty) result min)
-                         (csel (Cond.Lt) min result))))
-       result))
-
-(decl signed_min (Type) u64)
-(rule (signed_min $I8) -128)
-(rule (signed_min $I16) -32768)
-
-(decl signed_max (Type) u64)
-(rule (signed_max $I8) 127)
-(rule (signed_max $I16) 32767)
-
-(decl fpu_to_int (FpuToIntOp Reg) Reg)
-(rule (fpu_to_int op src)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.FpuToInt op dst src))))
-       dst))
-
-;; Helper for generating `MInst.IntToFpu` instructions.
-
-(decl int_to_fpu (IntToFpuOp Reg) Reg)
-(rule (int_to_fpu op src)
-      (let ((dst WritableReg (temp_writable_reg $I8X16))
-            (_ Unit (emit (MInst.IntToFpu op dst src))))
-       dst))
-
-;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput)
-(extern constructor gen_call gen_call)
-
-(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput)
-(extern constructor gen_call_indirect gen_call_indirect)
-
-;; Helpers for pinned register manipulation.
-
-(decl write_pinned_reg (Reg) SideEffectNoResult)
-(rule (write_pinned_reg val)
-      (mov_to_preg (preg_pinned) val))
-
-;; Helpers for stackslot effective address generation.
-
-(decl compute_stack_addr (StackSlot Offset32) Reg)
-(rule (compute_stack_addr stack_slot offset)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-           (_ Unit (emit (abi_stackslot_addr dst stack_slot offset))))
-        dst))
-
-;; Helper for emitting instruction sequences to perform a vector comparison.
-
-(decl vec_cmp_vc (Reg Reg VectorSize) Reg)
-(rule (vec_cmp_vc rn rm size)
-      (let ((dst Reg (vec_rrr (VecALUOp.Fcmeq) rn rn size))
-            (tmp Reg (vec_rrr (VecALUOp.Fcmeq) rm rm size))
-            (dst Reg (vec_rrr (VecALUOp.And) dst tmp size)))
-       dst))
-
-(decl vec_cmp (Reg Reg Type Cond) Reg)
-
-;; Floating point Vs / Vc
-(rule (vec_cmp rn rm ty (Cond.Vc))
-      (if (ty_vector_float ty))
-      (vec_cmp_vc rn rm (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Vs))
-      (if (ty_vector_float ty))
-      (let ((tmp Reg (vec_cmp_vc rn rm (vector_size ty))))
-       (vec_misc (VecMisc2.Not) tmp (vector_size ty))))
-
-;; 'Less than' operations are implemented by swapping the order of
-;; operands and using the 'greater than' instructions.
-;; 'Not equal' is implemented with 'equal' and inverting the result.
-
-;; Floating-point
-(rule (vec_cmp rn rm ty (Cond.Eq))
-      (if (ty_vector_float ty))
-      (vec_rrr (VecALUOp.Fcmeq) rn rm (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Ne))
-      (if (ty_vector_float ty))
-      (let ((tmp Reg (vec_rrr (VecALUOp.Fcmeq) rn rm (vector_size ty))))
-       (vec_misc (VecMisc2.Not) tmp (vector_size ty))))
-(rule (vec_cmp rn rm ty (Cond.Ge))
-      (if (ty_vector_float ty))
-      (vec_rrr (VecALUOp.Fcmge) rn rm (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Gt))
-      (if (ty_vector_float ty))
-      (vec_rrr (VecALUOp.Fcmgt) rn rm (vector_size ty)))
-;; Floating-point swapped-operands
-(rule (vec_cmp rn rm ty (Cond.Mi))
-      (if (ty_vector_float ty))
-      (vec_rrr (VecALUOp.Fcmgt) rm rn (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Ls))
-      (if (ty_vector_float ty))
-      (vec_rrr (VecALUOp.Fcmge) rm rn (vector_size ty)))
-
-;; Integer
-(rule 1 (vec_cmp rn rm ty (Cond.Eq))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmeq) rn rm (vector_size ty)))
-(rule 1 (vec_cmp rn rm ty (Cond.Ne))
-      (if (ty_vector_not_float ty))
-      (let ((tmp Reg (vec_rrr (VecALUOp.Cmeq) rn rm (vector_size ty))))
-       (vec_misc (VecMisc2.Not) tmp (vector_size ty))))
-(rule 1 (vec_cmp rn rm ty (Cond.Ge))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmge) rn rm (vector_size ty)))
-(rule 1 (vec_cmp rn rm ty (Cond.Gt))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmgt) rn rm (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Hs))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmhs) rn rm (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Hi))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmhi) rn rm (vector_size ty)))
-;; Integer swapped-operands
-(rule (vec_cmp rn rm ty (Cond.Le))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmge) rm rn (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Lt))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmgt) rm rn (vector_size ty)))
-(rule 1 (vec_cmp rn rm ty (Cond.Ls))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmhs) rm rn (vector_size ty)))
-(rule (vec_cmp rn rm ty (Cond.Lo))
-      (if (ty_vector_not_float ty))
-      (vec_rrr (VecALUOp.Cmhi) rm rn (vector_size ty)))
-
-;; Helper for determining if any value in a vector is true.
-;; This operation is implemented by using umaxp to create a scalar value, which
-;; is then compared against zero.
-;;
-;; umaxp vn.4s, vm.4s, vm.4s
-;; mov xm, vn.d[0]
-;; cmp xm, #0
-(decl vanytrue (Reg Type) ProducesFlags)
-(rule 1 (vanytrue src (ty_vec128 ty))
-      (let ((src Reg (vec_rrr (VecALUOp.Umaxp) src src (VectorSize.Size32x4)))
-            (src Reg (mov_from_vec src 0 (ScalarSize.Size64))))
-       (cmp_imm (OperandSize.Size64) src (u8_into_imm12 0))))
-(rule (vanytrue src ty)
-      (if (ty_vec64 ty))
-      (let ((src Reg (mov_from_vec src 0 (ScalarSize.Size64))))
-       (cmp_imm (OperandSize.Size64) src (u8_into_imm12 0))))
-
-;;;; TLS Values ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Helper for emitting ElfTlsGetAddr.
-(decl elf_tls_get_addr (ExternalName) Reg)
-(rule (elf_tls_get_addr name)
-      (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.ElfTlsGetAddr name dst))))
-        dst))
-
-;; A tuple of `ProducesFlags` and `IntCC`.
-(type FlagsAndCC (enum (FlagsAndCC (flags ProducesFlags)
-                                   (cc IntCC))))
-
-;; Helper constructor for `FlagsAndCC`.
-(decl flags_and_cc (ProducesFlags IntCC) FlagsAndCC)
-(rule (flags_and_cc flags cc) (FlagsAndCC.FlagsAndCC flags cc))
-
-;; Materialize a `FlagsAndCC` into a boolean `ValueRegs`.
-(decl flags_and_cc_to_bool (FlagsAndCC) ValueRegs)
-(rule (flags_and_cc_to_bool (FlagsAndCC.FlagsAndCC flags cc))
-      (with_flags flags (materialize_bool_result (cond_code cc))))
-
-;; Get the `ProducesFlags` out of a `FlagsAndCC`.
-(decl flags_and_cc_flags (FlagsAndCC) ProducesFlags)
-(rule (flags_and_cc_flags (FlagsAndCC.FlagsAndCC flags _cc)) flags)
-
-;; Get the `IntCC` out of a `FlagsAndCC`.
-(decl flags_and_cc_cc (FlagsAndCC) IntCC)
-(rule (flags_and_cc_cc (FlagsAndCC.FlagsAndCC _flags cc)) cc)
-
-;; Helpers for lowering `icmp` sequences.
-;; `lower_icmp` contains shared functionality for lowering `icmp`
-;; sequences, which `lower_icmp_into_{reg,flags}` extend from.
-(decl lower_icmp (IntCC Value Value Type) FlagsAndCC)
-(decl lower_icmp_into_reg (IntCC Value Value Type Type) ValueRegs)
-(decl lower_icmp_into_flags (IntCC Value Value Type) FlagsAndCC)
-(decl lower_icmp_const (IntCC Value u64 Type) FlagsAndCC)
-;; For most cases, `lower_icmp_into_flags` is the same as `lower_icmp`,
-;; except for some I128 cases (see below).
-(rule -1 (lower_icmp_into_flags cond x y ty) (lower_icmp cond x y ty))
-
-;; Vectors.
-;; `icmp` into flags for vectors is invalid.
-(rule 1 (lower_icmp_into_reg cond x y in_ty @ (multi_lane _ _) _out_ty)
-      (let ((cond Cond (cond_code cond))
-            (rn Reg (put_in_reg x))
-            (rm Reg (put_in_reg y)))
-       (vec_cmp rn rm in_ty cond)))
-
-;; Determines the appropriate extend op given the value type and whether it is signed.
-(decl lower_extend_op (Type bool) ExtendOp)
-(rule (lower_extend_op $I8 $true) (ExtendOp.SXTB))
-(rule (lower_extend_op $I16 $true) (ExtendOp.SXTH))
-(rule (lower_extend_op $I8 $false) (ExtendOp.UXTB))
-(rule (lower_extend_op $I16 $false) (ExtendOp.UXTH))
-
-;; Integers <= 64-bits.
-(rule -2 (lower_icmp_into_reg cond rn rm in_ty out_ty)
-      (if (ty_int_ref_scalar_64 in_ty))
-      (let ((cc Cond (cond_code cond)))
-        (flags_and_cc_to_bool (lower_icmp cond rn rm in_ty))))
-
-(rule 1 (lower_icmp cond rn rm (fits_in_16 ty))
-      (if (signed_cond_code cond))
-      (let ((rn Reg (put_in_reg_sext32 rn)))
-      (flags_and_cc (cmp_extend (operand_size ty) rn rm (lower_extend_op ty $true)) cond)))
-(rule -1 (lower_icmp cond rn (imm12_from_value rm) (fits_in_16 ty))
-      (let ((rn Reg (put_in_reg_zext32 rn)))
-      (flags_and_cc (cmp_imm (operand_size ty) rn rm) cond)))
-(rule -2 (lower_icmp cond rn rm (fits_in_16 ty))
-      (let ((rn Reg (put_in_reg_zext32 rn)))
-      (flags_and_cc (cmp_extend (operand_size ty) rn rm (lower_extend_op ty $false)) cond)))
-(rule -3 (lower_icmp cond rn (u64_from_iconst c) ty)
-      (if (ty_int_ref_scalar_64 ty))
-      (lower_icmp_const cond rn c ty))
-(rule -4 (lower_icmp cond rn rm ty)
-      (if (ty_int_ref_scalar_64 ty))
-      (flags_and_cc (cmp (operand_size ty) rn rm) cond))
-
-;; We get better encodings when testing against an immediate that's even instead
-;; of odd, so rewrite comparisons to use even immediates:
-;;
-;;         A >= B + 1
-;;     ==> A - 1 >= B
-;;     ==> A > B
-(rule (lower_icmp_const (IntCC.UnsignedGreaterThanOrEqual) a b ty)
-      (if (ty_int_ref_scalar_64 ty))
-      (if-let $true (u64_is_odd b))
-      (if-let (imm12_from_u64 imm) (u64_sub b 1))
-  (flags_and_cc (cmp_imm (operand_size ty) a imm) (IntCC.UnsignedGreaterThan)))
-(rule (lower_icmp_const (IntCC.SignedGreaterThanOrEqual) a b ty)
-      (if (ty_int_ref_scalar_64 ty))
-      (if-let $true (u64_is_odd b))
-      (if-let (imm12_from_u64 imm) (u64_sub b 1))
-  (flags_and_cc (cmp_imm (operand_size ty) a imm) (IntCC.SignedGreaterThan)))
-
-(rule -1 (lower_icmp_const cond rn (imm12_from_u64 c) ty)
-      (if (ty_int_ref_scalar_64 ty))
-  (flags_and_cc (cmp_imm (operand_size ty) rn c) cond))
-(rule -2 (lower_icmp_const cond rn c ty)
-      (if (ty_int_ref_scalar_64 ty))
-  (flags_and_cc (cmp (operand_size ty) rn (imm ty (ImmExtend.Zero) c)) cond))
-
-
-;; 128-bit integers.
-(rule (lower_icmp_into_reg cond @ (IntCC.Equal) rn rm $I128 $I8)
-      (let ((cc Cond (cond_code cond)))
-       (flags_and_cc_to_bool
-        (lower_icmp cond rn rm $I128))))
-(rule (lower_icmp_into_reg cond @ (IntCC.NotEqual) rn rm $I128 $I8)
-      (let ((cc Cond (cond_code cond)))
-       (flags_and_cc_to_bool
-        (lower_icmp cond rn rm $I128))))
-
-;; cmp lhs_lo, rhs_lo
-;; ccmp lhs_hi, rhs_hi, #0, eq
-(decl lower_icmp_i128_eq_ne (Value Value) ProducesFlags)
-(rule (lower_icmp_i128_eq_ne lhs rhs)
-      (let ((lhs ValueRegs (put_in_regs lhs))
-            (rhs ValueRegs (put_in_regs rhs))
-            (lhs_lo Reg (value_regs_get lhs 0))
-            (lhs_hi Reg (value_regs_get lhs 1))
-            (rhs_lo Reg (value_regs_get rhs 0))
-            (rhs_hi Reg (value_regs_get rhs 1))
-            (cmp_inst ProducesFlags (cmp (OperandSize.Size64) lhs_lo rhs_lo)))
-       (ccmp (OperandSize.Size64) lhs_hi rhs_hi
-        (nzcv $false $false $false $false) (Cond.Eq) cmp_inst)))
-
-(rule (lower_icmp (IntCC.Equal) lhs rhs $I128)
-      (flags_and_cc (lower_icmp_i128_eq_ne lhs rhs) (IntCC.Equal)))
-(rule (lower_icmp (IntCC.NotEqual) lhs rhs $I128)
-      (flags_and_cc (lower_icmp_i128_eq_ne lhs rhs) (IntCC.NotEqual)))
-
-;; cmp      lhs_lo, rhs_lo
-;; cset     tmp1, unsigned_cond
-;; cmp      lhs_hi, rhs_hi
-;; cset     tmp2, cond
-;; csel     dst, tmp1, tmp2, eq
-(rule -1 (lower_icmp_into_reg cond lhs rhs $I128 $I8)
-      (let ((unsigned_cond Cond (cond_code (intcc_unsigned cond)))
-            (cond Cond (cond_code cond))
-            (lhs ValueRegs (put_in_regs lhs))
-            (rhs ValueRegs (put_in_regs rhs))
-            (lhs_lo Reg (value_regs_get lhs 0))
-            (lhs_hi Reg (value_regs_get lhs 1))
-            (rhs_lo Reg (value_regs_get rhs 0))
-            (rhs_hi Reg (value_regs_get rhs 1))
-            (tmp1 Reg (with_flags_reg (cmp (OperandSize.Size64) lhs_lo rhs_lo)
-                                      (materialize_bool_result unsigned_cond))))
-        (with_flags (cmp (OperandSize.Size64) lhs_hi rhs_hi)
-                    (lower_icmp_i128_consumer cond tmp1))))
-
-(decl lower_icmp_i128_consumer (Cond Reg) ConsumesFlags)
-(rule (lower_icmp_i128_consumer cond tmp1)
-      (let ((tmp2 WritableReg (temp_writable_reg $I64))
-            (dst WritableReg (temp_writable_reg $I64)))
-       (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
-        (MInst.CSet tmp2 cond)
-        (MInst.CSel dst (Cond.Eq) tmp1 tmp2)
-        (value_reg dst))))
-
-(decl lower_bmask (Type Type ValueRegs) ValueRegs)
-
-
-;; For conversions that exactly fit a register, we can use csetm.
-;;
-;; cmp   val, #0
-;; csetm res, ne
-(rule 0
-      (lower_bmask (fits_in_64 _) (ty_32_or_64 in_ty) val)
-      (with_flags_reg
-        (cmp_imm (operand_size in_ty) (value_regs_get val 0) (u8_into_imm12 0))
-        (csetm (Cond.Ne))))
-
-;; For conversions from a 128-bit value into a 64-bit or smaller one, we or the
-;; two registers of the 128-bit value together, and then recurse with the
-;; combined value as a 64-bit test.
-;;
-;; orr   val, lo, hi
-;; cmp   val, #0
-;; csetm res, ne
-(rule 1
-      (lower_bmask (fits_in_64 ty) $I128 val)
-      (let ((lo Reg (value_regs_get val 0))
-            (hi Reg (value_regs_get val 1))
-            (combined Reg (orr $I64 lo hi)))
-        (lower_bmask ty $I64 (value_reg combined))))
-
-;; For converting from any type into i128, duplicate the result of
-;; converting to i64.
-(rule 2
-      (lower_bmask $I128 in_ty val)
-      (let ((res ValueRegs (lower_bmask $I64 in_ty val))
-            (res Reg (value_regs_get res 0)))
-        (value_regs res res)))
-
-;; For conversions smaller than a register, we need to mask off the high bits, and then
-;; we can recurse into the general case.
-;;
-;; and   tmp, val, #ty_mask
-;; cmp   tmp, #0
-;; csetm res, ne
-(rule 3
-      (lower_bmask out_ty (fits_in_16 in_ty) val)
-      (let ((mask_bits ImmLogic (imm_logic_from_u64 $I32 (ty_mask in_ty)))
-            (masked Reg (and_imm $I32 (value_regs_get val 0) mask_bits)))
-        (lower_bmask out_ty $I32 masked)))
-
-;; Exceptional `lower_icmp_into_flags` rules.
-;; We need to guarantee that the flags for `cond` are correct, so we
-;; compare `dst` with 1.
-(rule (lower_icmp_into_flags cond @ (IntCC.SignedGreaterThanOrEqual) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0))
-            (tmp Reg (imm $I64 (ImmExtend.Sign) 1))) ;; mov tmp, #1
-        (flags_and_cc (cmp (OperandSize.Size64) dst tmp) cond)))
-(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedGreaterThanOrEqual) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0))
-            (tmp Reg (imm $I64 (ImmExtend.Zero) 1)))
-        (flags_and_cc (cmp (OperandSize.Size64) dst tmp) cond)))
-(rule (lower_icmp_into_flags cond @ (IntCC.SignedLessThanOrEqual) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0))
-            (tmp Reg (imm $I64 (ImmExtend.Sign) 1)))
-       (flags_and_cc (cmp (OperandSize.Size64) tmp dst) cond)))
-(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedLessThanOrEqual) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0))
-            (tmp Reg (imm $I64 (ImmExtend.Zero) 1)))
-        (flags_and_cc (cmp (OperandSize.Size64) tmp dst) cond)))
-;; For strict comparisons, we compare with 0.
-(rule (lower_icmp_into_flags cond @ (IntCC.SignedGreaterThan) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0)))
-        (flags_and_cc (cmp (OperandSize.Size64) dst (zero_reg)) cond)))
-(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedGreaterThan) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0)))
-        (flags_and_cc (cmp (OperandSize.Size64) dst (zero_reg)) cond)))
-(rule (lower_icmp_into_flags cond @ (IntCC.SignedLessThan) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0)))
-       (flags_and_cc (cmp (OperandSize.Size64) (zero_reg) dst) cond)))
-(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedLessThan) lhs rhs $I128)
-      (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
-            (dst Reg (value_regs_get dst 0)))
-       (flags_and_cc (cmp (OperandSize.Size64) (zero_reg) dst) cond)))
-
-;; Helpers for generating select instruction sequences.
-(decl lower_select (ProducesFlags Cond Type Value Value) ValueRegs)
-(rule 2 (lower_select flags cond (ty_scalar_float ty) rn rm)
-      (with_flags flags (fpu_csel ty cond rn rm)))
-(rule 3 (lower_select flags cond (ty_vec128 ty) rn rm)
-      (with_flags flags (vec_csel cond rn rm)))
-(rule (lower_select flags cond ty rn rm)
-      (if (ty_vec64 ty))
-      (with_flags flags (fpu_csel $F64 cond rn rm)))
-(rule 4 (lower_select flags cond $I128 rn rm)
-      (let ((dst_lo WritableReg (temp_writable_reg $I64))
-            (dst_hi WritableReg (temp_writable_reg $I64))
-            (rn ValueRegs (put_in_regs rn))
-            (rm ValueRegs (put_in_regs rm))
-            (rn_lo Reg (value_regs_get rn 0))
-            (rn_hi Reg (value_regs_get rn 1))
-            (rm_lo Reg (value_regs_get rm 0))
-            (rm_hi Reg (value_regs_get rm 1)))
-       (with_flags flags
-        (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
-         (MInst.CSel dst_lo cond rn_lo rm_lo)
-         (MInst.CSel dst_hi cond rn_hi rm_hi)
-         (value_regs dst_lo dst_hi)))))
-(rule 1 (lower_select flags cond ty rn rm)
-      (if (ty_int_ref_scalar_64 ty))
-      (with_flags flags (csel cond rn rm)))
-
-;; Helper for emitting `MInst.Jump` instructions.
-(decl aarch64_jump (BranchTarget) SideEffectNoResult)
-(rule (aarch64_jump target)
-      (SideEffectNoResult.Inst (MInst.Jump target)))
-
-;; Helper for emitting `MInst.JTSequence` instructions.
-;; Emit the compound instruction that does:
-;;
-;; b.hs default
-;; csel rB, xzr, rIndex, hs
-;; csdb
-;; adr rA, jt
-;; ldrsw rB, [rA, rB, uxtw #2]
-;; add rA, rA, rB
-;; br rA
-;; [jt entries]
-;;
-;; This must be *one* instruction in the vcode because
-;; we cannot allow regalloc to insert any spills/fills
-;; in the middle of the sequence; otherwise, the ADR's
-;; PC-rel offset to the jumptable would be incorrect.
-;; (The alternative is to introduce a relocation pass
-;; for inlined jumptables, which is much worse, IMHO.)
-(decl jt_sequence (Reg BoxJTSequenceInfo) ConsumesFlags)
-(rule (jt_sequence ridx info)
-      (let ((rtmp1 WritableReg (temp_writable_reg $I64))
-            (rtmp2 WritableReg (temp_writable_reg $I64)))
-       (ConsumesFlags.ConsumesFlagsSideEffect
-        (MInst.JTSequence info ridx rtmp1 rtmp2))))
-
-;; Helper for emitting `MInst.CondBr` instructions.
-(decl cond_br (BranchTarget BranchTarget CondBrKind) ConsumesFlags)
-(rule (cond_br taken not_taken kind)
-      (ConsumesFlags.ConsumesFlagsSideEffect
-       (MInst.CondBr taken not_taken kind)))
-
-;; Helper for emitting `MInst.MovToNZCV` instructions.
-(decl mov_to_nzcv (Reg) ProducesFlags)
-(rule (mov_to_nzcv rn)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.MovToNZCV rn)))
-
-;; Helper for emitting `MInst.EmitIsland` instructions.
-(decl emit_island (CodeOffset) SideEffectNoResult)
-(rule (emit_island needed_space)
-      (SideEffectNoResult.Inst
-       (MInst.EmitIsland needed_space)))
-
-;; Helper for emitting `br_table` sequences.
-(decl br_table_impl (u64 Reg VecMachLabel) InstOutput)
-(rule (br_table_impl (imm12_from_u64 jt_size) ridx targets)
-      (let ((jt_info BoxJTSequenceInfo (targets_jt_info targets)))
-       (side_effect (with_flags_side_effect
-            (cmp_imm (OperandSize.Size32) ridx jt_size)
-            (jt_sequence ridx jt_info)))))
-(rule -1 (br_table_impl jt_size ridx targets)
-      (let ((jt_size Reg (imm $I64 (ImmExtend.Zero) jt_size))
-            (jt_info BoxJTSequenceInfo (targets_jt_info targets)))
-       (side_effect (with_flags_side_effect
-            (cmp (OperandSize.Size32) ridx jt_size)
-            (jt_sequence ridx jt_info)))))