wasmtime 0.4.1 → 5.0.0

data/ext/cargo-vendor/cranelift-codegen-0.91.0/src/isa/x64/inst.isle

@@ -1,3914 +0,0 @@
-;; Extern type definitions and constructors for the x64 `MachInst` type.
-
-;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Don't build `MInst` variants directly, in general. Instead, use the
-;; instruction-emitting helpers defined further down.
-
-(type MInst nodebug
-      (enum
-       ;; Nops of various sizes, including zero.
-       (Nop (len u8))
-
-       ;; =========================================
-       ;; Integer instructions.
-
-       ;; Integer arithmetic/bit-twiddling.
-       (AluRmiR (size OperandSize) ;; 4 or 8
-                (op AluRmiROpcode)
-                (src1 Gpr)
-                (src2 GprMemImm)
-                (dst WritableGpr))
-
-       ;; Integer arithmetic read-modify-write on memory.
-       (AluRM (size OperandSize) ;; 4 or 8
-              (op AluRmiROpcode)
-              (src1_dst SyntheticAmode)
-              (src2 Gpr))
-
-       ;; Instructions on general-purpose registers that only read src and
-       ;; defines dst (dst is not modified). `bsr`, etc.
-       (UnaryRmR (size OperandSize) ;; 2, 4, or 8
-                 (op UnaryRmROpcode)
-                 (src GprMem)
-                 (dst WritableGpr))
-
-       ;; Bitwise not.
-       (Not (size OperandSize) ;; 1, 2, 4, or 8
-            (src Gpr)
-            (dst WritableGpr))
-
-       ;; Integer negation.
-       (Neg (size OperandSize) ;; 1, 2, 4, or 8
-            (src Gpr)
-            (dst WritableGpr))
-
-       ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
-       (Div (size OperandSize) ;; 1, 2, 4, or 8
-            (signed bool)
-            (divisor GprMem)
-            (dividend_lo Gpr)
-            (dividend_hi Gpr)
-            (dst_quotient WritableGpr)
-            (dst_remainder WritableGpr))
-
-       ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX *
-       ;; rhs`.
-       (MulHi (size OperandSize)
-              (signed bool)
-              (src1 Gpr)
-              (src2 GprMem)
-              (dst_lo WritableGpr)
-              (dst_hi WritableGpr))
-
-       ;; A synthetic sequence to implement the right inline checks for
-       ;; remainder and division, assuming the dividend is in %rax.
-       ;;
-       ;; The generated code sequence is described in the emit's function match
-       ;; arm for this instruction.
-       (CheckedDivOrRemSeq (kind DivOrRemKind)
-                           (size OperandSize)
-                           (dividend_lo Gpr)
-                           (dividend_hi Gpr)
-                           (divisor Gpr)
-                           (dst_quotient WritableGpr)
-                           (dst_remainder WritableGpr)
-                           (tmp OptionWritableGpr))
-
-       ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd
-       ;; cdq cqo) or al into ah: (cbw)
-       (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8
-                       (src Gpr)
-                       (dst WritableGpr))
-
-       ;; Constant materialization: (imm32 imm64) reg.
-       ;;
-       ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32.
-       (Imm (dst_size OperandSize) ;; 4 or 8
-            (simm64 u64)
-            (dst WritableGpr))
-
-       ;; GPR to GPR move: mov (64 32) reg reg.
-       (MovRR (size OperandSize) ;; 4 or 8
-              (src Gpr)
-              (dst WritableGpr))
-
-       ;; Like `MovRR` but with a physical register source (for implementing
-       ;; CLIF instructions like `get_stack_pointer`).
-       (MovFromPReg (src PReg)
-                    (dst WritableGpr))
-
-       ;; Like `MovRR` but with a physical register destination (for
-       ;; implementing CLIF instructions like `set_pinned_reg`).
-       (MovToPReg (src Gpr)
-                  (dst PReg))
-
-       ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr
-       ;; reg.
-       ;;
-       ;; Note that the lq variant doesn't really exist since the default
-       ;; zero-extend rule makes it unnecessary. For that case we emit the
-       ;; equivalent "movl AM, reg32".
-       (MovzxRmR (ext_mode ExtMode)
-                 (src GprMem)
-                 (dst WritableGpr))
-
-       ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that.
-       (Mov64MR (src SyntheticAmode)
-                (dst WritableGpr))
-
-       ;; Loads the memory address of addr into dst.
-       (LoadEffectiveAddress (addr SyntheticAmode)
-                             (dst WritableGpr))
-
-       ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg.
-       (MovsxRmR (ext_mode ExtMode)
-                 (src GprMem)
-                 (dst WritableGpr))
-
-       ;; Integer stores: mov (b w l q) reg addr.
-       (MovRM (size OperandSize) ;; 1, 2, 4, or 8
-              (src Gpr)
-              (dst SyntheticAmode))
-
-       ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
-       (ShiftR (size OperandSize) ;; 1, 2, 4, or 8
-               (kind ShiftKind)
-               (src Gpr)
-               ;; shift count: `Imm8Gpr::Imm8(0 .. #bits-in-type - 1)` or
-               ;; `Imm8Reg::Gpr(r)` where `r` get's move mitosis'd into `%cl`.
-               (num_bits Imm8Gpr)
-               (dst WritableGpr))
-
-       ;; Arithmetic SIMD shifts.
-       (XmmRmiReg (opcode SseOpcode)
-                  (src1 Xmm)
-                  (src2 XmmMemImm)
-                  (dst WritableXmm))
-
-       ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
-       (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8
-                (opcode CmpOpcode)
-                (src GprMemImm)
-                (dst Gpr))
-
-       ;; Materializes the requested condition code in the destinaton reg.
-       (Setcc (cc CC)
-              (dst WritableGpr))
-
-       ;; Swaps byte order in register
-       (Bswap (size OperandSize) ;; 4 or 8
-              (src Gpr)
-              (dst WritableGpr))
-
-       ;; =========================================
-       ;; Conditional moves.
-
-       ;; GPR conditional move; overwrites the destination register.
-       (Cmove (size OperandSize)
-              (cc CC)
-              (consequent GprMem)
-              (alternative Gpr)
-              (dst WritableGpr))
-
-       ;; XMM conditional move; overwrites the destination register.
-       (XmmCmove (ty Type)
-                 (cc CC)
-                 (consequent XmmMem)
-                 (alternative Xmm)
-                 (dst WritableXmm))
-
-       ;; =========================================
-       ;; Stack manipulation.
-
-       ;; pushq (reg addr imm)
-       (Push64 (src GprMemImm))
-
-       ;; popq reg
-       (Pop64 (dst WritableGpr))
-
-      ;; Emits a inline stack probe loop.
-      (StackProbeLoop (tmp WritableReg)
-                      (frame_size u32)
-                      (guard_size u32))
-
-       ;; =========================================
-       ;; Floating-point operations.
-
-       ;; XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?)
-       ;; (32 64) (reg addr) reg
-       (XmmRmR (op SseOpcode)
-               (src1 Xmm)
-               (src2 XmmMem)
-               (dst WritableXmm))
-
-       ;; XMM (scalar or vector) blend op. The mask is used to blend between
-       ;; src1 and src2. This differs from a use of `XmmRmR` as the mask is
-       ;; implicitly in register xmm0; this special case exists to allow us to
-       ;; communicate the constraint on the `mask` register to regalloc2.
-       (XmmRmRBlend
-         (op SseOpcode)
-         (src1 Xmm)
-         (src2 XmmMem)
-         (mask Xmm)
-         (dst WritableXmm))
-
-       ;; XMM (scalar or vector) binary op that relies on the VEX prefix.
-       (XmmRmRVex (op AvxOpcode)
-                   (src1 Xmm)
-                   (src2 Xmm)
-                   (src3 XmmMem)
-                   (dst WritableXmm))
-
-       ;; XMM (scalar or vector) binary op that relies on the EVEX
-       ;; prefix. Takes two inputs.
-       (XmmRmREvex (op Avx512Opcode)
-                   (src1 XmmMem)
-                   (src2 Xmm)
-                   (dst WritableXmm))
-
-       ;; XMM (scalar or vector) binary op that relies on the EVEX
-       ;; prefix. Takes three inputs.
-       (XmmRmREvex3 (op Avx512Opcode)
-                   (src1 XmmMem)
-                   (src2 Xmm)
-                   (src3 Xmm)
-                   (dst WritableXmm))
-
-       ;; XMM (scalar or vector) unary op: mov between XMM registers (32 64)
-       ;; (reg addr) reg, sqrt, etc.
-       ;;
-       ;; This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is
-       ;; not used in the computation of the instruction dst value and so does
-       ;; not have to be a previously valid value. This is characteristic of mov
-       ;; instructions.
-       (XmmUnaryRmR (op SseOpcode)
-                    (src XmmMem)
-                    (dst WritableXmm))
-
-       ;; XMM (scalar or vector) unary op with immediate: roundss, roundsd, etc.
-       ;;
-       ;; This differs from XMM_RM_R_IMM in that the dst register of
-       ;; XmmUnaryRmRImm is not used in the computation of the instruction dst
-       ;; value and so does not have to be a previously valid value.
-       (XmmUnaryRmRImm (op SseOpcode)
-                       (src XmmMem)
-                       (imm u8)
-                       (dst WritableXmm))
-
-       ;; XMM (scalar or vector) unary op that relies on the EVEX prefix.
-       (XmmUnaryRmREvex (op Avx512Opcode)
-                        (src XmmMem)
-                        (dst WritableXmm))
-
-       ;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd,
-       ;; movq
-       (XmmMovRM (op SseOpcode)
-                 (src Reg)
-                 (dst SyntheticAmode))
-
-       ;; XMM (scalar) unary op (from xmm to integer reg): movd, movq,
-       ;; cvtts{s,d}2si
-       (XmmToGpr (op SseOpcode)
-                 (src Xmm)
-                 (dst WritableGpr)
-                 (dst_size OperandSize))
-
-       ;; XMM (scalar) unary op (from integer to float reg): movd, movq,
-       ;; cvtsi2s{s,d}
-       (GprToXmm (op SseOpcode)
-                 (src GprMem)
-                 (dst WritableXmm)
-                 (src_size OperandSize))
-
-       ;; Converts an unsigned int64 to a float32/float64.
-       (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8
-                            (src Gpr)
-                            (dst WritableXmm)
-                            (tmp_gpr1 WritableGpr)
-                            (tmp_gpr2 WritableGpr))
-
-       ;; Converts a scalar xmm to a signed int32/int64.
-       (CvtFloatToSintSeq (dst_size OperandSize)
-                          (src_size OperandSize)
-                          (is_saturating bool)
-                          (src Xmm)
-                          (dst WritableGpr)
-                          (tmp_gpr WritableGpr)
-                          (tmp_xmm WritableXmm))
-
-       ;; Converts a scalar xmm to an unsigned int32/int64.
-       (CvtFloatToUintSeq (dst_size OperandSize)
-                          (src_size OperandSize)
-                          (is_saturating bool)
-                          (src Xmm)
-                          (dst WritableGpr)
-                          (tmp_gpr WritableGpr)
-                          (tmp_xmm WritableXmm)
-                          (tmp_xmm2 WritableXmm))
-
-       ;; A sequence to compute min/max with the proper NaN semantics for xmm
-       ;; registers.
-       (XmmMinMaxSeq (size OperandSize)
-                     (is_min bool)
-                     (lhs Xmm)
-                     (rhs Xmm)
-                     (dst WritableXmm))
-
-       ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
-       (XmmCmpRmR (op SseOpcode)
-                  (src XmmMem)
-                  (dst Xmm))
-
-       ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm
-       ;; (reg addr) reg
-       ;;
-       ;; Note: this has to use `Reg*`, not `Xmm*`, operands because it is used
-       ;; in various lane insertion and extraction instructions that move
-       ;; between XMMs and GPRs.
-       (XmmRmRImm (op SseOpcode)
-                  (src1 Reg)
-                  (src2 RegMem)
-                  (dst WritableReg)
-                  (imm u8)
-                  (size OperandSize))
-
-       ;; =========================================
-       ;; Control flow instructions.
-
-       ;; Direct call: call simm32.
-       (CallKnown (dest ExternalName)
-                  (info BoxCallInfo))
-
-       ;; Indirect call: callq (reg mem)
-       (CallUnknown (dest RegMem)
-                    (info BoxCallInfo))
-
-       ;; A pseudo-instruction that captures register arguments in vregs.
-       (Args
-        (args VecArgPair))
-
-       ;; Return.
-       (Ret (rets VecRetPair))
-
-       ;; Jump to a known target: jmp simm32.
-       (JmpKnown (dst MachLabel))
-
-       ;; One-way conditional branch: jcond cond target.
-       ;;
-       ;; This instruction is useful when we have conditional jumps depending on
-       ;; more than two conditions, see for instance the lowering of Brz/brnz
-       ;; with Fcmp inputs.
-       ;;
-       ;; A note of caution: in contexts where the branch target is another
-       ;; block, this has to be the same successor as the one specified in the
-       ;; terminator branch of the current block.  Otherwise, this might confuse
-       ;; register allocation by creating new invisible edges.
-       (JmpIf (cc CC)
-              (taken MachLabel))
-
-       ;; Two-way conditional branch: jcond cond target target.
-       ;;
-       ;; Emitted as a compound sequence; the MachBuffer will shrink it as
-       ;; appropriate.
-       (JmpCond (cc CC)
-                (taken MachLabel)
-                (not_taken MachLabel))
-
-       ;; Jump-table sequence, as one compound instruction (see note in lower.rs
-       ;; for rationale).
-       ;;
-       ;; The generated code sequence is described in the emit's function match
-       ;; arm for this instruction.
-       ;;
-       ;; See comment on jmp_table_seq below about the temporaries signedness.
-       (JmpTableSeq (idx Reg)
-                    (tmp1 WritableReg)
-                    (tmp2 WritableReg)
-                    (default_target MachLabel)
-                    (targets BoxVecMachLabel))
-
-       ;; Indirect jump: jmpq (reg mem).
-       (JmpUnknown (target RegMem))
-
-       ;; Traps if the condition code is set.
-       (TrapIf (cc CC)
-               (trap_code TrapCode))
-
-       ;; Traps if both of the condition codes are set.
-       (TrapIfAnd (cc1 CC)
-                  (cc2 CC)
-                  (trap_code TrapCode))
-
-       ;; Traps if either of the condition codes are set.
-       (TrapIfOr (cc1 CC)
-                 (cc2 CC)
-                 (trap_code TrapCode))
-
-       ;; A debug trap.
-       (Hlt)
-
-       ;; An instruction that will always trigger the illegal instruction
-       ;; exception.
-       (Ud2 (trap_code TrapCode))
-
-       ;; Loads an external symbol in a register, with a relocation:
-       ;;
-       ;; movq $name@GOTPCREL(%rip), dst    if PIC is enabled, or
-       ;; movabsq $name, dst                otherwise.
-       (LoadExtName (dst WritableReg)
-                    (name BoxExternalName)
-                    (offset i64))
-
-       ;; =========================================
-       ;; Instructions pertaining to atomic memory accesses.
-
-       ;; A standard (native) `lock cmpxchg src, (amode)`, with register
-       ;; conventions:
-       ;;
-       ;; `mem`          (read) address
-       ;; `replacement`  (read) replacement value
-       ;; %rax           (modified) in: expected value, out: value that was actually at `dst`
-       ;; %rflags is written.  Do not assume anything about it after the instruction.
-       ;;
-       ;; The instruction "succeeded" iff the lowest `ty` bits of %rax
-       ;; afterwards are the same as they were before.
-       (LockCmpxchg (ty Type) ;; I8, I16, I32, or I64
-                    (replacement Reg)
-                    (expected Reg)
-                    (mem SyntheticAmode)
-                    (dst_old WritableReg))
-
-       ;; A synthetic instruction, based on a loop around a native `lock
-       ;; cmpxchg` instruction.
-       ;;
-       ;; This atomically modifies a value in memory and returns the old value.
-       ;; The sequence consists of an initial "normal" load from `dst`, followed
-       ;; by a loop which computes the new value and tries to compare-and-swap
-       ;; ("CAS") it into `dst`, using the native instruction `lock
-       ;; cmpxchg{b,w,l,q}`.  The loop iterates until the CAS is successful. If
-       ;; there is no contention, there will be only one pass through the loop
-       ;; body.  The sequence does *not* perform any explicit memory fence
-       ;; instructions (`mfence`/`sfence`/`lfence`).
-       ;;
-       ;; Note that the transaction is atomic in the sense that, as observed by
-       ;; some other thread, `dst` either has the initial or final value, but no
-       ;; other.  It isn't atomic in the sense of guaranteeing that no other
-       ;; thread writes to `dst` in between the initial load and the CAS -- but
-       ;; that would cause the CAS to fail unless the other thread's last write
-       ;; before the CAS wrote the same value that was already there.  In other
-       ;; words, this implementation suffers (unavoidably) from the A-B-A
-       ;; problem.
-       ;;
-       ;; This instruction sequence has fixed register uses as follows:
-       ;; - %rax  (written) the old value at `mem`
-       ;; - %rflags is written.  Do not assume anything about it after the
-       ;;   instruction.
-       (AtomicRmwSeq (ty Type) ;; I8, I16, I32, or I64
-                     (op MachAtomicRmwOp)
-                     (mem SyntheticAmode)
-                     (operand Reg)
-                     (temp WritableReg)
-                     (dst_old WritableReg))
-
-       ;; A memory fence (mfence, lfence or sfence).
-       (Fence (kind FenceKind))
-
-       ;; =========================================
-       ;; Meta-instructions generating no code.
-
-       ;; Marker, no-op in generated code: SP "virtual offset" is adjusted.
-       ;;
-       ;; This controls how `MemArg::NominalSPOffset` args are lowered.
-       (VirtualSPOffsetAdj (offset i64))
-
-       ;; Provides a way to tell the register allocator that the upcoming
-       ;; sequence of instructions will overwrite `dst` so it should be
-       ;; considered as a `def`; use this with care.
-       ;;
-       ;; This is useful when we have a sequence of instructions whose register
-       ;; usages are nominally `mod`s, but such that the combination of
-       ;; operations creates a result that is independent of the initial
-       ;; register value. It's thus semantically a `def`, not a `mod`, when all
-       ;; the instructions are taken together, so we want to ensure the register
-       ;; is defined (its live-range starts) prior to the sequence to keep
-       ;; analyses happy.
-       ;;
-       ;; One alternative would be a compound instruction that somehow
-       ;; encapsulates the others and reports its own `def`s/`use`s/`mod`s; this
-       ;; adds complexity (the instruction list is no longer flat) and requires
-       ;; knowledge about semantics and initial-value independence anyway.
-       (XmmUninitializedValue (dst WritableXmm))
-
-       ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol
-       ;; `dst`, which is constrained to `rax`.
-       (ElfTlsGetAddr (symbol ExternalName)
-                      (dst WritableGpr))
-
-       ;; A Mach-O TLS symbol access. Returns address of the TLS symbol in
-       ;; `dst`, which is constrained to `rax`.
-       (MachOTlsGetAddr (symbol ExternalName)
-                        (dst WritableGpr))
-
-       ;; A Coff TLS symbol access. Returns address of the TLS symbol in
-       ;; `dst`, which is constrained to `rax`.
-       (CoffTlsGetAddr (symbol ExternalName)
-                       (dst WritableGpr)
-                       (tmp WritableGpr))
-
-       ;; An unwind pseudoinstruction describing the state of the machine at
-       ;; this program point.
-       (Unwind (inst UnwindInst))
-
-       ;; A pseudoinstruction that just keeps a value alive.
-       (DummyUse (reg Reg))))
-
-(type OperandSize extern
-      (enum Size8
-            Size16
-            Size32
-            Size64))
-
-(type FenceKind extern
-      (enum MFence
-            LFence
-            SFence))
-
-(type BoxCallInfo extern (enum))
-
-(type BoxVecMachLabel extern (enum))
-
-(type MachLabelSlice extern (enum))
-
-;; The size of the jump table.
-(decl jump_table_size (BoxVecMachLabel) u32)
-(extern constructor jump_table_size jump_table_size)
-
-;; Extract a the target from a MachLabelSlice with exactly one target.
-(decl single_target (MachLabel) MachLabelSlice)
-(extern extractor single_target single_target)
-
-;; Extract a the targets from a MachLabelSlice with exactly two targets.
-(decl two_targets (MachLabel MachLabel) MachLabelSlice)
-(extern extractor two_targets two_targets)
-
-;; Extract the default target and jump table from a MachLabelSlice.
-(decl jump_table_targets (MachLabel BoxVecMachLabel) MachLabelSlice)
-(extern extractor jump_table_targets jump_table_targets)
-
-;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits.
-(decl operand_size_of_type_32_64 (Type) OperandSize)
-(extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64)
-
-;; Get the true `OperandSize` for a given `Type`, with no rounding.
-(decl raw_operand_size_of_type (Type) OperandSize)
-(extern constructor raw_operand_size_of_type raw_operand_size_of_type)
-
-;; Get the bit width of an `OperandSize`.
-(decl operand_size_bits (OperandSize) u16)
-(rule (operand_size_bits (OperandSize.Size8)) 8)
-(rule (operand_size_bits (OperandSize.Size16)) 16)
-(rule (operand_size_bits (OperandSize.Size32)) 32)
-(rule (operand_size_bits (OperandSize.Size64)) 64)
-
-(type AluRmiROpcode extern
-      (enum Add
-            Adc
-            Sub
-            Sbb
-            And
-            Or
-            Xor
-            Mul))
-
-(type UnaryRmROpcode extern
-      (enum Bsr
-            Bsf
-            Lzcnt
-            Tzcnt
-            Popcnt))
-
-(type DivOrRemKind extern
-      (enum SignedDiv
-            UnsignedDiv
-            SignedRem
-            UnsignedRem))
-
-(type SseOpcode extern
-      (enum Addps
-            Addpd
-            Addss
-            Addsd
-            Andps
-            Andpd
-            Andnps
-            Andnpd
-            Blendvpd
-            Blendvps
-            Comiss
-            Comisd
-            Cmpps
-            Cmppd
-            Cmpss
-            Cmpsd
-            Cvtdq2ps
-            Cvtdq2pd
-            Cvtpd2ps
-            Cvtps2pd
-            Cvtsd2ss
-            Cvtsd2si
-            Cvtsi2ss
-            Cvtsi2sd
-            Cvtss2si
-            Cvtss2sd
-            Cvttpd2dq
-            Cvttps2dq
-            Cvttss2si
-            Cvttsd2si
-            Divps
-            Divpd
-            Divss
-            Divsd
-            Insertps
-            Maxps
-            Maxpd
-            Maxss
-            Maxsd
-            Minps
-            Minpd
-            Minss
-            Minsd
-            Movaps
-            Movapd
-            Movd
-            Movdqa
-            Movdqu
-            Movlhps
-            Movmskps
-            Movmskpd
-            Movq
-            Movss
-            Movsd
-            Movups
-            Movupd
-            Mulps
-            Mulpd
-            Mulss
-            Mulsd
-            Orps
-            Orpd
-            Pabsb
-            Pabsw
-            Pabsd
-            Packssdw
-            Packsswb
-            Packusdw
-            Packuswb
-            Paddb
-            Paddd
-            Paddq
-            Paddw
-            Paddsb
-            Paddsw
-            Paddusb
-            Paddusw
-            Palignr
-            Pand
-            Pandn
-            Pavgb
-            Pavgw
-            Pblendvb
-            Pcmpeqb
-            Pcmpeqw
-            Pcmpeqd
-            Pcmpeqq
-            Pcmpgtb
-            Pcmpgtw
-            Pcmpgtd
-            Pcmpgtq
-            Pextrb
-            Pextrw
-            Pextrd
-            Pinsrb
-            Pinsrw
-            Pinsrd
-            Pmaddubsw
-            Pmaddwd
-            Pmaxsb
-            Pmaxsw
-            Pmaxsd
-            Pmaxub
-            Pmaxuw
-            Pmaxud
-            Pminsb
-            Pminsw
-            Pminsd
-            Pminub
-            Pminuw
-            Pminud
-            Pmovmskb
-            Pmovsxbd
-            Pmovsxbw
-            Pmovsxbq
-            Pmovsxwd
-            Pmovsxwq
-            Pmovsxdq
-            Pmovzxbd
-            Pmovzxbw
-            Pmovzxbq
-            Pmovzxwd
-            Pmovzxwq
-            Pmovzxdq
-            Pmuldq
-            Pmulhw
-            Pmulhuw
-            Pmulhrsw
-            Pmulld
-            Pmullw
-            Pmuludq
-            Por
-            Pshufb
-            Pshufd
-            Psllw
-            Pslld
-            Psllq
-            Psraw
-            Psrad
-            Psrlw
-            Psrld
-            Psrlq
-            Psubb
-            Psubd
-            Psubq
-            Psubw
-            Psubsb
-            Psubsw
-            Psubusb
-            Psubusw
-            Ptest
-            Punpckhbw
-            Punpckhwd
-            Punpcklbw
-            Punpcklwd
-            Pxor
-            Rcpss
-            Roundps
-            Roundpd
-            Roundss
-            Roundsd
-            Rsqrtss
-            Shufps
-            Sqrtps
-            Sqrtpd
-            Sqrtss
-            Sqrtsd
-            Subps
-            Subpd
-            Subss
-            Subsd
-            Ucomiss
-            Ucomisd
-            Unpcklps
-            Xorps
-            Xorpd))
-
-(type CmpOpcode extern
-      (enum Cmp
-            Test))
-
-(type RegMemImm extern
-      (enum
-       (Reg (reg Reg))
-       (Mem (addr SyntheticAmode))
-       (Imm (simm32 u32))))
-
-;; Put the given clif value into a `RegMemImm` operand.
-;;
-;; Asserts that the value fits into a single register, and doesn't require
-;; multiple registers for its representation (like `i128` for example).
-;;
-;; As a side effect, this marks the value as used.
-(decl put_in_reg_mem_imm (Value) RegMemImm)
-(extern constructor put_in_reg_mem_imm put_in_reg_mem_imm)
-
-(type RegMem extern
-      (enum
-       (Reg (reg Reg))
-       (Mem (addr SyntheticAmode))))
-
-;; Convert a RegMem to a RegMemImm.
-(decl reg_mem_to_reg_mem_imm (RegMem) RegMemImm)
-(rule (reg_mem_to_reg_mem_imm (RegMem.Reg reg))
-      (RegMemImm.Reg reg))
-(rule (reg_mem_to_reg_mem_imm (RegMem.Mem addr))
-      (RegMemImm.Mem addr))
-
-;; Put the given clif value into a `RegMem` operand.
-;;
-;; Asserts that the value fits into a single register, and doesn't require
-;; multiple registers for its representation (like `i128` for example).
-;;
-;; As a side effect, this marks the value as used.
-(decl put_in_reg_mem (Value) RegMem)
-(extern constructor put_in_reg_mem put_in_reg_mem)
-
-;; Addressing modes.
-
-(type SyntheticAmode extern (enum))
-
-(decl synthetic_amode_to_reg_mem (SyntheticAmode) RegMem)
-(extern constructor synthetic_amode_to_reg_mem synthetic_amode_to_reg_mem)
-
-(decl amode_to_synthetic_amode (Amode) SyntheticAmode)
-(extern constructor amode_to_synthetic_amode amode_to_synthetic_amode)
-
-;; An `Amode` represents a possible addressing mode that can be used
-;; in instructions. These denote a 64-bit value only.
-(type Amode (enum
-             ;; Immediate sign-extended and a register
-             (ImmReg (simm32 u32)
-                     (base Reg)
-                     (flags MemFlags))
-
-             ;; Sign-extend-32-to-64(simm32) + base + (index << shift)
-             (ImmRegRegShift (simm32 u32)
-                             (base Gpr)
-                             (index Gpr)
-                             (shift u8)
-                             (flags MemFlags))
-
-             ;; Sign-extend-32-to-64(immediate) + RIP (instruction
-             ;; pointer). The appropriate relocation is emitted so
-             ;; that the resulting immediate makes this Amode refer to
-             ;; the given MachLabel.
-             (RipRelative (target MachLabel))))
-
-;; Some Amode constructor helpers.
-
-(decl amode_with_flags (Amode MemFlags) Amode)
-(extern constructor amode_with_flags amode_with_flags)
-
-(decl amode_imm_reg (u32 Gpr) Amode)
-(extern constructor amode_imm_reg amode_imm_reg)
-
-(decl amode_imm_reg_flags (u32 Gpr MemFlags) Amode)
-(rule (amode_imm_reg_flags offset base flags)
-      (amode_with_flags (amode_imm_reg offset base) flags))
-
-(decl amode_imm_reg_reg_shift (u32 Gpr Gpr u8) Amode)
-(extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift)
-
-(decl amode_imm_reg_reg_shift_flags (u32 Gpr Gpr u8 MemFlags) Amode)
-(rule (amode_imm_reg_reg_shift_flags offset base index shift flags)
-      (amode_with_flags (amode_imm_reg_reg_shift offset base index shift) flags))
-
-;; A helper to both check that the `Imm64` and `Offset32` values sum to less
-;; than 32-bits AND return this summed `u32` value. Also, the `Imm64` will be
-;; zero-extended from `Type` up to 64 bits. This is useful for `to_amode`.
-(decl pure sum_extend_fits_in_32_bits (Type Imm64 Offset32) u32)
-(extern constructor sum_extend_fits_in_32_bits sum_extend_fits_in_32_bits)
-
-;;;; Amode lowering ;;;;
-
-;; To generate an address for a memory access, we can pattern-match
-;; various CLIF sub-trees to x64's complex addressing modes (`Amode`).
-;;
-;; Information about available addressing modes is available in
-;; Intel's Software Developer's Manual, volume 2, section 2.1.5,
-;; "Addressing-Mode Encoding of ModR/M and SIB Bytes."
-;;
-;; The general strategy to build an `Amode` is to traverse over the
-;; input expression's addends, recursively deconstructing a tree of
-;; `iadd` operators that add up parts of the address, updating the
-;; `Amode` in an incremental fashion as we add in each piece.
-;;
-;; We start with an "immediate + register" form that encapsulates the
-;; load/store's built-in `Offset32` and `invalid_reg` as the
-;; register. This is given by `amode_initial`. Then we add `Value`s
-;; one at a time with `amode_add`. (Why start with `invalid_reg` at
-;; all? Because we don't want to special-case the first input and
-;; duplicate rules; this lets us use the "add a value" logic even for
-;; the first value.)
-;;
-;; It is always valid to use `amode_add` to add the one single
-;; `address` input to the load/store (i.e., the `Value` given to
-;; `to_amode`). In the fallback case, this is what we do. Then we get
-;; an `Amode.ImmReg` with the `Offset32` and `Value` below and nothing
-;; else; this always works and is not *that* bad.
-;;
-;; But we can often do better. The toplevel rule for `iadd` below will
-;; turn an `(amode_add amode (iadd a b))` into two invocations of
-;; `amode_add`, for each operand of the `iadd`. This is what allows us
-;; to handle sums of many parts.
-;;
-;; Then we "just" need to work out how we can incorporate a new
-;; component into an existing addressing mode:
-;;
-;; - Case 1: When we have an `ImmReg` and the register is
-;;   `invalid_reg` (the initial `Amode` above), we can put the new
-;;   addend into a register and insert it into the `ImmReg`.
-;;
-;; - Case 2: When we have an `ImmReg` with a valid register already,
-;;   and we have another register to add, we can transition to an
-;;   `ImmRegRegShift`.
-;;
-;; - Case 3: When we're adding an `ishl`, we can refine the above rule
-;;   and use the built-in multiplier of 1, 2, 4, 8 to implement a
-;;   left-shift by 0, 1, 2, 3.
-;;
-;; - Case 4: When we are adding another constant offset, we can fold
-;;   it into the existing offset, as long as the sum still fits into
-;;   the signed 32-bit field.
-;;
-;; - Case 5: And as a general fallback, we can generate a new `add`
-;;   instruction and add the new addend to an existing component of
-;;   the `Amode`.
-(decl to_amode (MemFlags Value Offset32) Amode)
-
-;; Initial step in amode processing: create an ImmReg with
-;; (invalid_reg) and encapsulating the flags and offset from the
-;; load/store.
-(decl amode_initial (MemFlags Offset32) Amode)
-(rule (amode_initial flags (offset32 off))
-      (Amode.ImmReg off (invalid_reg) flags))
-
-;; One step in amode processing: take an existing amode and add
-;; another value to it.
-(decl amode_add (Amode Value) Amode)
-
-;; -- Top-level driver: pull apart the addends.
-;;
-;; Any amode can absorb an `iadd` by absorbing first the LHS of the
-;; add, then the RHS.
-;;
-;; Priority 2 to take this above fallbacks and ensure we traverse the
-;; `iadd` tree fully.
-(rule 2 (amode_add amode (iadd x y))
-      (let ((amode1 Amode (amode_add amode x))
-            (amode2 Amode (amode_add amode1 y)))
-        amode2))
-
-;; -- Case 1 (adding a register to the initial Amode with invalid_reg).
-;;
-;; An Amode.ImmReg with invalid_reg (initial state) can absorb a
-;; register as the base register.
-(rule (amode_add (Amode.ImmReg off (invalid_reg) flags) value)
-      (Amode.ImmReg off value flags))
-
-;; -- Case 2 (adding a register to an Amode with a register already).
-;;
-;; An Amode.ImmReg can absorb another register as the index register.
-(rule (amode_add (Amode.ImmReg off (valid_reg base) flags) value)
-      ;; Shift of 0 --> base + 1*value.
-      (Amode.ImmRegRegShift off base value 0 flags))
-
-;; -- Case 3 (adding a shifted value to an Amode).
-;;
-;; An Amode.ImmReg can absorb a shift of another register as the index register.
-;;
-;; Priority 2 to take these rules above generic case.
-(rule 2 (amode_add (Amode.ImmReg off (valid_reg base) flags) (ishl index (iconst (uimm8 shift))))
-      (if (u32_lteq (u8_as_u32 shift) 3))
-      (Amode.ImmRegRegShift off base index shift flags))
-(rule 2 (amode_add (Amode.ImmReg off (valid_reg base) flags) (uextend (ishl index (iconst (uimm8 shift)))))
-      (if (u32_lteq (u8_as_u32 shift) 3))
-      (Amode.ImmRegRegShift off base (extend_to_gpr index $I64 (ExtendKind.Zero)) shift flags))
-
-;; Same, but with a uextend of a shift of a 32-bit add. This is valid
-;; because we know our lowering of a narrower-than-64-bit `iadd` will
-;; always write the full register width, so we can effectively ignore
-;; the `uextend` and look through it to the `ishl`.
-;;
-;; Priority 3 to avoid conflict with the previous rule.
-(rule 3 (amode_add (Amode.ImmReg off (valid_reg base) flags)
-                   (uextend (ishl index @ (iadd _ _) (iconst (uimm8 shift)))))
-      (if (u32_lteq (u8_as_u32 shift) 3))
-      (Amode.ImmRegRegShift off base index shift flags))
-
-;; -- Case 4 (absorbing constant offsets).
-;;
-;; An Amode can absorb a constant (i64, or extended i32) as long as
-;; the sum still fits in the signed-32-bit offset.
-;;
-;; Priority 3 in order to take this option above the fallback
-;; (immediate in register). Two rules, for imm+reg and
-;; imm+reg+scale*reg cases.
-(rule 3 (amode_add (Amode.ImmReg off base flags)
-                   (iconst (simm32 c)))
-      (if-let sum (s32_add_fallible off c))
-      (Amode.ImmReg sum base flags))
-(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
-                   (iconst (simm32 c)))
-      (if-let sum (s32_add_fallible off c))
-      (Amode.ImmRegRegShift sum base index shift flags))
-
-;; Likewise for a zero-extended i32 const, as long as the constant
-;; wasn't negative. (Why nonnegative? Because adding a
-;; non-sign-extended negative to a 64-bit address is not the same as
-;; adding in simm32-space.)
-(rule 3 (amode_add (Amode.ImmReg off base flags)
-                   (uextend (iconst (simm32 (u32_nonnegative c)))))
-      (if-let sum (s32_add_fallible off c))
-      (Amode.ImmReg sum base flags))
-(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
-                   (uextend (iconst (simm32 (u32_nonnegative c)))))
-      (if-let sum (s32_add_fallible off c))
-      (Amode.ImmRegRegShift sum base index shift flags))
-
-;; Likewise for a sign-extended i32 const.
-(rule 3 (amode_add (Amode.ImmReg off base flags)
-                   (sextend (iconst (simm32 c))))
-      (if-let sum (s32_add_fallible off c))
-      (Amode.ImmReg sum base flags))
-(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
-                   (sextend (iconst (simm32 c))))
-      (if-let sum (s32_add_fallible off c))
-      (Amode.ImmRegRegShift sum base index shift flags))
-
-;; -- Case 5 (fallback to add a new value to an imm+reg+scale*reg).
-;;
-;; An Amode.ImmRegRegShift can absorb any other value by creating a
-;; new add instruction and replacing the base with
-;; (base+value).
-(rule (amode_add (Amode.ImmRegRegShift off base index shift flags) value)
-      (let ((sum Gpr (x64_add $I64 base value)))
-        (Amode.ImmRegRegShift off sum index shift flags)))
-
-;; Finally, define the toplevel `to_amode`.
-(rule (to_amode flags base offset)
-      (amode_finalize (amode_add (amode_initial flags offset) base)))
-
-;; If an amode has no registers at all and only offsets (a constant
-;; value), we need to "finalize" it by sticking in a zero'd reg in
-;; place of the (invalid_reg) produced by (amode_initial).
-(decl amode_finalize (Amode) Amode)
-(rule 1 (amode_finalize (Amode.ImmReg off (invalid_reg) flags))
-      (Amode.ImmReg off (imm $I64 0) flags))
-(rule 0 (amode_finalize amode)
-      amode)
-
-;; Offsetting an Amode. Used when we need to do consecutive
-;; loads/stores to adjacent addresses.
-(decl amode_offset (Amode u32) Amode)
-(extern constructor amode_offset amode_offset)
-
-;; Return a zero offset as an `Offset32`.
-(decl zero_offset () Offset32)
-(extern constructor zero_offset zero_offset)
-
-;; Shift kinds.
-
-(type ShiftKind extern
-      (enum ShiftLeft
-            ShiftRightLogical
-            ShiftRightArithmetic
-            RotateLeft
-            RotateRight))
-
-(type Imm8Reg extern
-      (enum (Imm8 (imm u8))
-            (Reg (reg Reg))))
-
-;; Put the given clif value into a `Imm8Reg` operand, masked to the bit width of
-;; the given type.
-;;
-;; Asserts that the value fits into a single register, and doesn't require
-;; multiple registers for its representation (like `i128` for example).
-;;
-;; As a side effect, this marks the value as used.
-;;
-;; This is used when lowering various shifts and rotates.
-(decl put_masked_in_imm8_gpr (Value Type) Imm8Gpr)
-(rule 2 (put_masked_in_imm8_gpr (u64_from_iconst amt) ty)
-      (const_to_type_masked_imm8 amt ty))
-(rule 1 (put_masked_in_imm8_gpr amt (fits_in_16 ty))
-      (x64_and $I64 (value_regs_get_gpr amt 0) (RegMemImm.Imm (shift_mask ty))))
-(rule (put_masked_in_imm8_gpr amt ty)
-      (value_regs_get_gpr amt 0))
-
-;; Condition codes
-(type CC extern
-      (enum O
-            NO
-            B
-            NB
-            Z
-            NZ
-            BE
-            NBE
-            S
-            NS
-            L
-            NL
-            LE
-            NLE
-            P
-            NP))
-
-(decl intcc_to_cc (IntCC) CC)
-(extern constructor intcc_to_cc intcc_to_cc)
-
-(decl cc_invert (CC) CC)
-(extern constructor cc_invert cc_invert)
-
-;; Fails if the argument is not either CC.NZ or CC.Z.
-(decl cc_nz_or_z (CC) CC)
-(extern extractor cc_nz_or_z cc_nz_or_z)
-
-(type AvxOpcode extern
-      (enum Vfmadd213ss
-            Vfmadd213sd
-            Vfmadd213ps
-            Vfmadd213pd))
-
-(type Avx512Opcode extern
-      (enum Vcvtudq2ps
-            Vpabsq
-            Vpermi2b
-            Vpmullq
-            Vpopcntb))
-
-(type FcmpImm extern
-      (enum Equal
-            LessThan
-            LessThanOrEqual
-            Unordered
-            NotEqual
-            UnorderedOrGreaterThanOrEqual
-            UnorderedOrGreaterThan
-            Ordered))
-
-(decl encode_fcmp_imm (FcmpImm) u8)
-(extern constructor encode_fcmp_imm encode_fcmp_imm)
-
-(type RoundImm extern
-      (enum RoundNearest
-            RoundDown
-            RoundUp
-            RoundZero))
-
-(decl encode_round_imm (RoundImm) u8)
-(extern constructor encode_round_imm encode_round_imm)
-
-;;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(type Gpr (primitive Gpr))
-(type WritableGpr (primitive WritableGpr))
-(type OptionWritableGpr (primitive OptionWritableGpr))
-(type GprMem extern (enum))
-(type GprMemImm extern (enum))
-(type Imm8Gpr extern (enum))
-
-(type Xmm (primitive Xmm))
-(type WritableXmm (primitive WritableXmm))
-(type OptionWritableXmm (primitive OptionWritableXmm))
-(type XmmMem extern (enum))
-(type XmmMemImm extern (enum))
-
-;; Convert an `Imm8Reg` into an `Imm8Gpr`.
-(decl imm8_reg_to_imm8_gpr (Imm8Reg) Imm8Gpr)
-(extern constructor imm8_reg_to_imm8_gpr imm8_reg_to_imm8_gpr)
-
-;; Convert a `WritableGpr` to a `WritableReg`.
-(decl writable_gpr_to_reg (WritableGpr) WritableReg)
-(extern constructor writable_gpr_to_reg writable_gpr_to_reg)
-
-;; Convert a `WritableXmm` to a `WritableReg`.
-(decl writable_xmm_to_reg (WritableXmm) WritableReg)
-(extern constructor writable_xmm_to_reg writable_xmm_to_reg)
-
-;; Convert a `WritableReg` to a `WritableXmm`.
-(decl writable_reg_to_xmm (WritableReg) WritableXmm)
-(extern constructor writable_reg_to_xmm writable_reg_to_xmm)
-
-;; Convert a `WritableXmm` to an `Xmm`.
-(decl writable_xmm_to_xmm (WritableXmm) Xmm)
-(extern constructor writable_xmm_to_xmm writable_xmm_to_xmm)
-
-;; Convert a `WritableGpr` to an `Gpr`.
-(decl writable_gpr_to_gpr (WritableGpr) Gpr)
-(extern constructor writable_gpr_to_gpr writable_gpr_to_gpr)
-
-;; Convert an `Gpr` to a `Reg`.
-(decl gpr_to_reg (Gpr) Reg)
-(extern constructor gpr_to_reg gpr_to_reg)
-
-;; Convert an `Gpr` to a `GprMem`.
-(decl gpr_to_gpr_mem (Gpr) GprMem)
-(extern constructor gpr_to_gpr_mem gpr_to_gpr_mem)
-
-;; Convert an `Gpr` to a `GprMemImm`.
-(decl gpr_to_gpr_mem_imm (Gpr) GprMemImm)
-(extern constructor gpr_to_gpr_mem_imm gpr_to_gpr_mem_imm)
-
-;; Convert an `Xmm` to a `Reg`.
-(decl xmm_to_reg (Xmm) Reg)
-(extern constructor xmm_to_reg xmm_to_reg)
-
-;; Convert an `Xmm` into an `XmmMemImm`.
-(decl xmm_to_xmm_mem_imm (Xmm) XmmMemImm)
-(extern constructor xmm_to_xmm_mem_imm xmm_to_xmm_mem_imm)
-
-;; Allocate a new temporary GPR register.
-(decl temp_writable_gpr () WritableGpr)
-(extern constructor temp_writable_gpr temp_writable_gpr)
-
-;; Allocate a new temporary XMM register.
-(decl temp_writable_xmm () WritableXmm)
-(extern constructor temp_writable_xmm temp_writable_xmm)
-
-;; Construct a new `XmmMem` from the given `RegMem`.
-;;
-;; Asserts that the `RegMem`'s register, if any, is an XMM register.
-(decl reg_mem_to_xmm_mem (RegMem) XmmMem)
-(extern constructor reg_mem_to_xmm_mem reg_mem_to_xmm_mem)
-
-;; Construct a new `RegMemImm` from the given `Reg`.
-(decl reg_to_reg_mem_imm (Reg) RegMemImm)
-(extern constructor reg_to_reg_mem_imm reg_to_reg_mem_imm)
-
-;; Construct a new `GprMemImm` from the given `RegMemImm`.
-;;
-;; Asserts that the `RegMemImm`'s register, if any, is an GPR register.
-(decl gpr_mem_imm_new (RegMemImm) GprMemImm)
-(extern constructor gpr_mem_imm_new gpr_mem_imm_new)
-
-;; Construct a new `XmmMemImm` from the given `RegMemImm`.
-;;
-;; Asserts that the `RegMemImm`'s register, if any, is an XMM register.
-(decl xmm_mem_imm_new (RegMemImm) XmmMemImm)
-(extern constructor xmm_mem_imm_new xmm_mem_imm_new)
-
-;; Construct a new `XmmMem` from an `Xmm`.
-(decl xmm_to_xmm_mem (Xmm) XmmMem)
-(extern constructor xmm_to_xmm_mem xmm_to_xmm_mem)
-
-;; Construct a new `XmmMem` from an `RegMem`.
-(decl xmm_mem_to_reg_mem (XmmMem) RegMem)
-(extern constructor xmm_mem_to_reg_mem xmm_mem_to_reg_mem)
-
-;; Convert a `GprMem` to a `RegMem`.
-(decl gpr_mem_to_reg_mem (GprMem) RegMem)
-(extern constructor gpr_mem_to_reg_mem gpr_mem_to_reg_mem)
-
-;; Construct a new `Xmm` from a `Reg`.
-;;
-;; Asserts that the register is a XMM.
-(decl xmm_new (Reg) Xmm)
-(extern constructor xmm_new xmm_new)
-
-;; Construct a new `Gpr` from a `Reg`.
-;;
-;; Asserts that the register is a GPR.
-(decl gpr_new (Reg) Gpr)
-(extern constructor gpr_new gpr_new)
-
-;; Construct a new `GprMem` from a `RegMem`.
-;;
-;; Asserts that the `RegMem`'s register, if any, is a GPR.
-(decl reg_mem_to_gpr_mem (RegMem) GprMem)
-(extern constructor reg_mem_to_gpr_mem reg_mem_to_gpr_mem)
-
-;; Construct a `GprMem` from a `Reg`.
-;;
-;; Asserts that the `Reg` is a GPR.
-(decl reg_to_gpr_mem (Reg) GprMem)
-(extern constructor reg_to_gpr_mem reg_to_gpr_mem)
-
-;; Construct a `GprMemImm` from a `Reg`.
-;;
-;; Asserts that the `Reg` is a GPR.
-(decl reg_to_gpr_mem_imm (Reg) GprMemImm)
-(rule (reg_to_gpr_mem_imm r)
-      (gpr_to_gpr_mem_imm (gpr_new r)))
-
-;; Put a value into a GPR.
-;;
-;; Asserts that the value goes into a GPR.
-(decl put_in_gpr (Value) Gpr)
-(rule (put_in_gpr val)
-      (gpr_new (put_in_reg val)))
-
-;; Put a value into a `GprMem`.
-;;
-;; Asserts that the value goes into a GPR.
-(decl put_in_gpr_mem (Value) GprMem)
-(rule (put_in_gpr_mem val)
-      (reg_mem_to_gpr_mem (put_in_reg_mem val)))
-
-;; Put a value into a `GprMemImm`.
-;;
-;; Asserts that the value goes into a GPR.
-(decl put_in_gpr_mem_imm (Value) GprMemImm)
-(rule (put_in_gpr_mem_imm val)
-      (gpr_mem_imm_new (put_in_reg_mem_imm val)))
-
-;; Put a value into a XMM.
-;;
-;; Asserts that the value goes into a XMM.
-(decl put_in_xmm (Value) Xmm)
-(rule (put_in_xmm val)
-      (xmm_new (put_in_reg val)))
-
-;; Put a value into a `XmmMem`.
-;;
-;; Asserts that the value goes into a XMM.
-(decl put_in_xmm_mem (Value) XmmMem)
-(extern constructor put_in_xmm_mem put_in_xmm_mem)
-
-;; Put a value into a `XmmMemImm`.
-;;
-;; Asserts that the value goes into a XMM.
-(decl put_in_xmm_mem_imm (Value) XmmMemImm)
-(extern constructor put_in_xmm_mem_imm put_in_xmm_mem_imm)
-
-;; Construct an `InstOutput` out of a single GPR register.
-(decl output_gpr (Gpr) InstOutput)
-(rule (output_gpr x)
-      (output_reg (gpr_to_reg x)))
-
-;; Construct a `ValueRegs` out of two GPR registers.
-(decl value_gprs (Gpr Gpr) ValueRegs)
-(rule (value_gprs x y)
-      (value_regs (gpr_to_reg x) (gpr_to_reg y)))
-
-;; Construct an `InstOutput` out of a single XMM register.
-(decl output_xmm (Xmm) InstOutput)
-(rule (output_xmm x)
-      (output_reg (xmm_to_reg x)))
-
-;; Get the `n`th reg in a `ValueRegs` and construct a GPR from it.
-;;
-;; Asserts that the register is a GPR.
-(decl value_regs_get_gpr (ValueRegs usize) Gpr)
-(rule (value_regs_get_gpr regs n)
-      (gpr_new (value_regs_get regs n)))
-
-;; Convert a `Gpr` to an `Imm8Gpr`.
-(decl gpr_to_imm8_gpr (Gpr) Imm8Gpr)
-(extern constructor gpr_to_imm8_gpr gpr_to_imm8_gpr)
-
-;; Convert an 8-bit immediate into an `Imm8Gpr`.
-(decl imm8_to_imm8_gpr (u8) Imm8Gpr)
-(extern constructor imm8_to_imm8_gpr imm8_to_imm8_gpr)
-
-;; Get the low half of the given `Value` as a GPR.
-(decl lo_gpr (Value) Gpr)
-(rule (lo_gpr regs) (gpr_new (lo_reg regs)))
-
-;;;; Helpers for Working With Integer Comparison Codes ;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
-
-;; This is a direct import of `IntCC::without_equal`.
-;; Get the corresponding IntCC with the equal component removed.
-;; For conditions without a zero component, this is a no-op.
-(decl intcc_without_eq (IntCC) IntCC)
-(extern constructor intcc_without_eq intcc_without_eq)
-
-;;;; Helpers for determining the register class of a value type ;;;;;;;;;;;;;;;;
-
-(type RegisterClass
-      (enum
-        (Gpr (single_register bool))
-        (Xmm)))
-
-(decl type_register_class (RegisterClass) Type)
-(extern extractor type_register_class type_register_class)
-
-(decl is_xmm_type (Type) Type)
-(extractor (is_xmm_type ty) (and (type_register_class (RegisterClass.Xmm)) ty))
-
-(decl is_gpr_type (Type) Type)
-(extractor (is_gpr_type ty) (and (type_register_class (RegisterClass.Gpr _)) ty))
-
-(decl is_single_register_gpr_type (Type) Type)
-(extractor (is_single_register_gpr_type ty)
-           (and (type_register_class (RegisterClass.Gpr $true)) ty))
-
-(decl is_multi_register_gpr_type (Type) Type)
-(extractor (is_multi_register_gpr_type ty)
-           (and (type_register_class (RegisterClass.Gpr $false)) ty))
-
-;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl avx512vl_enabled (bool) Type)
-(extern extractor infallible avx512vl_enabled avx512vl_enabled)
-
-(decl avx512dq_enabled (bool) Type)
-(extern extractor infallible avx512dq_enabled avx512dq_enabled)
-
-(decl avx512f_enabled (bool) Type)
-(extern extractor infallible avx512f_enabled avx512f_enabled)
-
-(decl avx512bitalg_enabled (bool) Type)
-(extern extractor infallible avx512bitalg_enabled avx512bitalg_enabled)
-
-(decl avx512vbmi_enabled (bool) Type)
-(extern extractor infallible avx512vbmi_enabled avx512vbmi_enabled)
-
-(decl use_lzcnt (bool) Type)
-(extern extractor infallible use_lzcnt use_lzcnt)
-
-(decl use_bmi1 (bool) Type)
-(extern extractor infallible use_bmi1 use_bmi1)
-
-(decl use_popcnt (bool) Type)
-(extern extractor infallible use_popcnt use_popcnt)
-
-(decl use_fma (bool) Type)
-(extern extractor infallible use_fma use_fma)
-
-(decl use_sse41 (bool) Type)
-(extern extractor infallible use_sse41 use_sse41)
-
-;;;; Helpers for Merging and Sinking Immediates/Loads  ;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Extract a constant `Imm8Reg.Imm8` from a value operand.
-(decl imm8_from_value (Imm8Reg) Value)
-(extern extractor imm8_from_value imm8_from_value)
-
-;; Mask a constant to the bit-width of the given type and package it into an
-;; `Imm8Reg.Imm8`. This is used for shifts and rotates, so that we don't try and
-;; shift/rotate more bits than the type has available, per Cranelift's
-;; semantics.
-(decl const_to_type_masked_imm8 (u64 Type) Imm8Gpr)
-(extern constructor const_to_type_masked_imm8 const_to_type_masked_imm8)
-
-;; Generate a mask for the bit-width of the given type
-(decl shift_mask (Type) u32)
-(extern constructor shift_mask shift_mask)
-
-;; Extract a constant `GprMemImm.Imm` from a value operand.
-(decl simm32_from_value (GprMemImm) Value)
-(extern extractor simm32_from_value simm32_from_value)
-
-;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate.
-(decl simm32_from_imm64 (GprMemImm) Imm64)
-(extern extractor simm32_from_imm64 simm32_from_imm64)
-
-;; A load that can be sunk into another operation.
-(type SinkableLoad extern (enum))
-
-;; Extract a `SinkableLoad` that works with `RegMemImm.Mem` from a value
-;; operand.
-(decl sinkable_load (SinkableLoad) Value)
-(extern extractor sinkable_load sinkable_load)
-
-;; Sink a `SinkableLoad` into a `RegMemImm.Mem`.
-;;
-;; This is a side-effectful operation that notifies the context that the
-;; instruction that produced the `SinkableImm` has been sunk into another
-;; instruction, and no longer needs to be lowered.
-(decl sink_load (SinkableLoad) RegMem)
-(extern constructor sink_load sink_load)
-
-(decl sink_load_to_gpr_mem_imm (SinkableLoad) GprMemImm)
-(rule (sink_load_to_gpr_mem_imm load)
-      (gpr_mem_imm_new (sink_load load)))
-
-(decl sink_load_to_xmm_mem (SinkableLoad) XmmMem)
-(rule (sink_load_to_xmm_mem load)
-      (reg_mem_to_xmm_mem (sink_load load)))
-
-;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(type ExtKind extern
-      (enum None
-            SignExtend
-            ZeroExtend))
-
-(type ExtendKind (enum Sign Zero))
-
-(type ExtMode extern (enum BL BQ WL WQ LQ))
-
-;; `ExtMode::new`
-(decl ext_mode (u16 u16) ExtMode)
-(extern constructor ext_mode ext_mode)
-
-;; Put the given value into a register, but extended as the given type.
-(decl extend_to_gpr (Value Type ExtendKind) Gpr)
-
-;; If the value is already of the requested type, no extending is necessary.
-;;
-;; Priority 1 because the equality constraint doesn't prove that this rule
-;; doesn't overlap with the one below.
-(rule 1 (extend_to_gpr (and val (value_type ty)) ty _kind)
-      (put_in_gpr val))
-
-(rule (extend_to_gpr (and val (value_type from_ty))
-                     to_ty
-                     kind)
-      (let ((from_bits u16 (ty_bits_u16 from_ty))
-            ;; Use `operand_size_of_type` so that the we clamp the output to 32-
-            ;; or 64-bit width types.
-            (to_bits u16 (operand_size_bits (operand_size_of_type_32_64 to_ty))))
-        (extend kind
-                to_ty
-                (ext_mode from_bits to_bits)
-                (put_in_gpr_mem val))))
-
-;; Do a sign or zero extension of the given `GprMem`.
-(decl extend (ExtendKind Type ExtMode GprMem) Gpr)
-
-;; Zero extending uses `movzx`.
-(rule (extend (ExtendKind.Zero) ty mode src)
-      (x64_movzx mode src))
-
-;; Sign extending uses `movsx`.
-(rule (extend (ExtendKind.Sign) ty mode src)
-      (x64_movsx mode src))
-
-;;;; Helpers for Working SSE tidbits ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Turn a vector type into its integer-typed vector equivalent.
-(decl vec_int_type (Type) Type)
-(rule (vec_int_type (multi_lane 8 16)) $I8X16)
-(rule (vec_int_type (multi_lane 16 8)) $I16X8)
-(rule (vec_int_type (multi_lane 32 4)) $I32X4)
-(rule (vec_int_type (multi_lane 64 2)) $I64X2)
-
-;; Determine the appropriate operation for xor-ing vectors of the specified type
-(decl sse_xor_op (Type) SseOpcode)
-(rule 1 (sse_xor_op $F32X4) (SseOpcode.Xorps))
-(rule 1 (sse_xor_op $F64X2) (SseOpcode.Xorpd))
-(rule 1 (sse_xor_op $F32) (SseOpcode.Xorps))
-(rule 1 (sse_xor_op $F64) (SseOpcode.Xorpd))
-
-;; Priority 0 because multi_lane overlaps with the previous two explicit type
-;; patterns.
-(rule 0 (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor))
-
-;; Performs an xor operation of the two operands specified.
-(decl sse_xor (Type Xmm XmmMem) Xmm)
-(rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y))
-
-;; Generates a register value which has an all-ones pattern.
-;;
-;; Note that this is accomplished by comparing a fresh register with itself,
-;; which for integers is always true. Also note that the comparison is always
-;; done for integers. This is because we're comparing a fresh register to itself
-;; and we don't know the previous contents of the register. If a floating-point
-;; comparison is used then it runs the risk of comparing NaN against NaN and not
-;; actually producing an all-ones mask. By using integer comparision operations
-;; we're guaranteeed that everything is equal to itself.
-(decl vector_all_ones () Xmm)
-(rule (vector_all_ones)
-      (let ((r WritableXmm (temp_writable_xmm)))
-        (x64_pcmpeqd r r)))
-
-;; Helper for creating XmmUninitializedValue instructions.
-(decl xmm_uninit_value () Xmm)
-(rule (xmm_uninit_value)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUninitializedValue dst))))
-        dst))
-
-;; Helper for creating an SSE register holding an `i64x2` from two `i64` values.
-(decl make_i64x2_from_lanes (GprMem GprMem) Xmm)
-(rule (make_i64x2_from_lanes lo hi)
-      (let ((dst Xmm (xmm_uninit_value))
-            (dst Xmm (x64_pinsrd dst lo 0 (OperandSize.Size64)))
-            (dst Xmm (x64_pinsrd dst hi 1 (OperandSize.Size64))))
-        dst))
-
-;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary.
-(decl mov_rmi_to_xmm (RegMemImm) XmmMemImm)
-(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Mem _)) (xmm_mem_imm_new rmi))
-(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Imm _)) (xmm_mem_imm_new rmi))
-(rule (mov_rmi_to_xmm (RegMemImm.Reg r))
-      (gpr_to_xmm (SseOpcode.Movd)
-                  r
-                  (OperandSize.Size32)))
-
-;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput)
-(extern constructor gen_call gen_call)
-
-(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput)
-(extern constructor gen_call_indirect gen_call_indirect)
-
-;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Helper for constructing a LoadExtName instruction.
-(decl load_ext_name (ExternalName i64) Reg)
-(rule (load_ext_name extname offset)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.LoadExtName dst extname offset))))
-        dst))
-
-;; Load a value into a register.
-(decl x64_load (Type SyntheticAmode ExtKind) Reg)
-
-(rule 1 (x64_load (fits_in_32 ty) addr (ExtKind.SignExtend))
-      (x64_movsx (ext_mode (ty_bytes ty) 8)
-             addr))
-
-(rule 2 (x64_load $I64 addr _ext_kind)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.Mov64MR addr dst))))
-        dst))
-
-(rule 2 (x64_load $F32 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movss)
-                      addr))
-
-(rule 2 (x64_load $F64 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movsd)
-                      addr))
-
-(rule 2 (x64_load $F32X4 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movups)
-                      addr))
-
-(rule 2 (x64_load $F64X2 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movupd)
-                      addr))
-
-(rule 0 (x64_load (multi_lane _bits _lanes) addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movdqu) addr))
-
-(decl x64_mov (Amode) Reg)
-(rule (x64_mov addr)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.Mov64MR addr dst))))
-        dst))
-
-(decl x64_movzx (ExtMode GprMem) Gpr)
-(rule (x64_movzx mode src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.MovzxRmR mode src dst))))
-        dst))
-
-(decl x64_movsx (ExtMode GprMem) Gpr)
-(rule (x64_movsx mode src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.MovsxRmR mode src dst))))
-        dst))
-
-(decl x64_movss_load (XmmMem) Xmm)
-(rule (x64_movss_load from)
-      (xmm_unary_rm_r (SseOpcode.Movss) from))
-
-(decl x64_movsd_load (XmmMem) Xmm)
-(rule (x64_movsd_load from)
-      (xmm_unary_rm_r (SseOpcode.Movsd) from))
-
-(decl x64_movups (XmmMem) Xmm)
-(rule (x64_movups from)
-      (xmm_unary_rm_r (SseOpcode.Movups) from))
-
-(decl x64_movupd (XmmMem) Xmm)
-(rule (x64_movupd from)
-      (xmm_unary_rm_r (SseOpcode.Movupd) from))
-
-(decl x64_movd (Xmm) Gpr)
-(rule (x64_movd from)
-      (xmm_to_gpr (SseOpcode.Movd) from (OperandSize.Size32)))
-
-(decl x64_movdqu (XmmMem) Xmm)
-(rule (x64_movdqu from)
-      (xmm_unary_rm_r (SseOpcode.Movdqu) from))
-
-(decl x64_movapd (XmmMem) Xmm)
-(rule (x64_movapd src)
-      (xmm_unary_rm_r (SseOpcode.Movapd) src))
-
-(decl x64_pmovsxbw (XmmMem) Xmm)
-(rule (x64_pmovsxbw from)
-      (xmm_unary_rm_r (SseOpcode.Pmovsxbw) from))
-
-(decl x64_pmovzxbw (XmmMem) Xmm)
-(rule (x64_pmovzxbw from)
-      (xmm_unary_rm_r (SseOpcode.Pmovzxbw) from))
-
-(decl x64_pmovsxwd (XmmMem) Xmm)
-(rule (x64_pmovsxwd from)
-      (xmm_unary_rm_r (SseOpcode.Pmovsxwd) from))
-
-(decl x64_pmovzxwd (XmmMem) Xmm)
-(rule (x64_pmovzxwd from)
-      (xmm_unary_rm_r (SseOpcode.Pmovzxwd) from))
-
-(decl x64_pmovsxdq (XmmMem) Xmm)
-(rule (x64_pmovsxdq from)
-      (xmm_unary_rm_r (SseOpcode.Pmovsxdq) from))
-
-(decl x64_pmovzxdq (XmmMem) Xmm)
-(rule (x64_pmovzxdq from)
-      (xmm_unary_rm_r (SseOpcode.Pmovzxdq) from))
-
-(decl x64_movrm (Type SyntheticAmode Gpr) SideEffectNoResult)
-(rule (x64_movrm ty addr data)
-      (let ((size OperandSize (raw_operand_size_of_type ty)))
-        (SideEffectNoResult.Inst (MInst.MovRM size data addr))))
-
-(decl x64_xmm_movrm (SseOpcode SyntheticAmode Xmm) SideEffectNoResult)
-(rule (x64_xmm_movrm op addr data)
-      (SideEffectNoResult.Inst (MInst.XmmMovRM op data addr)))
-
-;; Load a constant into an XMM register.
-(decl x64_xmm_load_const (Type VCodeConstant) Xmm)
-(rule (x64_xmm_load_const ty const)
-      (x64_load ty (const_to_synthetic_amode const) (ExtKind.None)))
-
-;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
-;; These constructors create SSA-style `MInst`s. It is their responsibility to
-;; maintain the invariant that each temporary register they allocate and define
-;; only gets defined the once.
-
-;; Helper for emitting `MInst.AluRmiR` instructions.
-(decl alu_rmi_r (Type AluRmiROpcode Gpr GprMemImm) Gpr)
-(rule (alu_rmi_r ty opcode src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst))))
-        dst))
-
-;; Helper for emitting `add` instructions.
-(decl x64_add (Type Gpr GprMemImm) Gpr)
-(rule (x64_add ty src1 src2)
-      (alu_rmi_r ty
-                 (AluRmiROpcode.Add)
-                 src1
-                 src2))
-
-;; Helper for creating `add` instructions whose flags are also used.
-(decl x64_add_with_flags_paired (Type Gpr GprMemImm) ProducesFlags)
-(rule (x64_add_with_flags_paired ty src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr)))
-        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
-         (MInst.AluRmiR (operand_size_of_type_32_64 ty)
-                        (AluRmiROpcode.Add)
-                        src1
-                        src2
-                        dst)
-         dst)))
-
-;; Helper for creating `adc` instructions.
-(decl x64_adc_paired (Type Gpr GprMemImm) ConsumesFlags)
-(rule (x64_adc_paired ty src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr)))
-        (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
-         (MInst.AluRmiR (operand_size_of_type_32_64 ty)
-                        (AluRmiROpcode.Adc)
-                        src1
-                        src2
-                        dst)
-         dst)))
-
-;; Helper for emitting `sub` instructions.
-(decl x64_sub (Type Gpr GprMemImm) Gpr)
-(rule (x64_sub ty src1 src2)
-      (alu_rmi_r ty
-                 (AluRmiROpcode.Sub)
-                 src1
-                 src2))
-
-;; Helper for creating `sub` instructions whose flags are also used.
-(decl x64_sub_with_flags_paired (Type Gpr GprMemImm) ProducesFlags)
-(rule (x64_sub_with_flags_paired ty src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr)))
-        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
-         (MInst.AluRmiR (operand_size_of_type_32_64 ty)
-                        (AluRmiROpcode.Sub)
-                        src1
-                        src2
-                        dst)
-         dst)))
-
-;; Helper for creating `sbb` instructions.
-(decl x64_sbb_paired (Type Gpr GprMemImm) ConsumesFlags)
-(rule (x64_sbb_paired ty src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr)))
-        (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
-         (MInst.AluRmiR (operand_size_of_type_32_64 ty)
-                        (AluRmiROpcode.Sbb)
-                        src1
-                        src2
-                        dst)
-         dst)))
-
-;; Helper for creating `mul` instructions.
-(decl x64_mul (Type Gpr GprMemImm) Gpr)
-(rule (x64_mul ty src1 src2)
-      (alu_rmi_r ty
-                 (AluRmiROpcode.Mul)
-                 src1
-                 src2))
-
-;; Helper for emitting `and` instructions.
-(decl x64_and (Type Gpr GprMemImm) Gpr)
-(rule (x64_and ty src1 src2)
-      (alu_rmi_r ty
-                 (AluRmiROpcode.And)
-                 src1
-                 src2))
-
-(decl x64_and_with_flags_paired (Type Gpr GprMemImm) ProducesFlags)
-(rule (x64_and_with_flags_paired ty src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr)))
-           (ProducesFlags.ProducesFlagsSideEffect
-                 (MInst.AluRmiR (operand_size_of_type_32_64 ty)
-                       (AluRmiROpcode.And)
-                       src1
-                       src2
-                       dst))))
-
-;; Helper for emitting `or` instructions.
-(decl x64_or (Type Gpr GprMemImm) Gpr)
-(rule (x64_or ty src1 src2)
-      (alu_rmi_r ty
-                 (AluRmiROpcode.Or)
-                 src1
-                 src2))
-
-;; Helper for emitting `xor` instructions.
-(decl x64_xor (Type Gpr GprMemImm) Gpr)
-(rule (x64_xor ty src1 src2)
-      (alu_rmi_r ty
-                 (AluRmiROpcode.Xor)
-                 src1
-                 src2))
-
-;; Helper for emitting immediates with an `i64` value. Note that
-;; integer constants in ISLE are always parsed as `i128`s; this enables
-;; negative numbers to be used as immediates.
-(decl imm_i64 (Type i64) Reg)
-(rule (imm_i64 ty value)
-      (imm ty (i64_as_u64 value)))
-
-(decl nonzero_u64_fits_in_u32 (u64) u64)
-(extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32)
-
-;; Helper for emitting immediates.
-;;
-;; There are three priorities in use in this rule:
-;; 2 - rules that match on an explicit type
-;; 1 - rules that match on types that fit in 64 bits
-;; 0 - rules that match on vectors
-(decl imm (Type u64) Reg)
-
-;; Integer immediates.
-(rule 1 (imm (fits_in_64 ty) (u64_nonzero simm64))
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.Imm size simm64 dst))))
-        dst))
-
-;; `f32` immediates.
-(rule 2 (imm $F32 (u64_nonzero bits))
-      (gpr_to_xmm (SseOpcode.Movd)
-                  (imm $I32 bits)
-                  (OperandSize.Size32)))
-
-;; `f64` immediates.
-(rule 2 (imm $F64 (u64_nonzero bits))
-      (gpr_to_xmm (SseOpcode.Movq)
-                  (imm $I64 bits)
-                  (OperandSize.Size64)))
-
-;; Special case for when a 64-bit immediate fits into 32-bits. We can use a
-;; 32-bit move that zero-extends the value, which has a smaller encoding.
-(rule 2 (imm $I64 (nonzero_u64_fits_in_u32 x))
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst))))
-        dst))
-
-;; Special case for integer zero immediates: turn them into an `xor r, r`.
-(rule 1 (imm (fits_in_64 ty) (u64_zero))
-      (let ((wgpr WritableGpr (temp_writable_gpr))
-            (g Gpr wgpr)
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.AluRmiR size
-                                         (AluRmiROpcode.Xor)
-                                         g
-                                         g
-                                         wgpr))))
-        (gpr_to_reg g)))
-
-;; Special case for zero immediates with vector types, they turn into an xor
-;; specific to the vector type.
-(rule 0 (imm ty @ (multi_lane _bits _lanes) 0)
-      (let ((wr WritableXmm (temp_writable_xmm))
-            (r Xmm wr)
-            (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty)
-                                        r
-                                        r
-                                        wr))))
-        (xmm_to_reg r)))
-
-;; Special case for `f32` zero immediates to use `xorps`.
-(rule 2 (imm $F32 (u64_zero))
-      (let ((wr WritableXmm (temp_writable_xmm))
-            (r Xmm wr)
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorps)
-                                        r
-                                        r
-                                        wr))))
-        (xmm_to_reg r)))
-
-;; TODO: use cmpeqps for all 1s
-
-;; Special case for `f64` zero immediates to use `xorpd`.
-(rule 2 (imm $F64 (u64_zero))
-      (let ((wr WritableXmm (temp_writable_xmm))
-            (r Xmm wr)
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorpd)
-                                        r
-                                        r
-                                        wr))))
-        (xmm_to_reg r)))
-
-;; TODO: use cmpeqpd for all 1s
-
-;; Helper for creating `MInst.ShiftR` instructions.
-(decl shift_r (Type ShiftKind Gpr Imm8Gpr) Gpr)
-(rule (shift_r ty kind src1 src2)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            ;; Use actual 8/16-bit instructions when appropriate: we
-            ;; rely on their shift-amount-masking semantics.
-            (size OperandSize (raw_operand_size_of_type ty))
-            (_ Unit (emit (MInst.ShiftR size kind src1 src2 dst))))
-        dst))
-
-;; Helper for creating `rotl` instructions.
-(decl x64_rotl (Type Gpr Imm8Gpr) Gpr)
-(rule (x64_rotl ty src1 src2)
-      (shift_r ty (ShiftKind.RotateLeft) src1 src2))
-
-;; Helper for creating `rotr` instructions.
-(decl x64_rotr (Type Gpr Imm8Gpr) Gpr)
-(rule (x64_rotr ty src1 src2)
-      (shift_r ty (ShiftKind.RotateRight) src1 src2))
-
-;; Helper for creating `shl` instructions.
-(decl x64_shl (Type Gpr Imm8Gpr) Gpr)
-(rule (x64_shl ty src1 src2)
-      (shift_r ty (ShiftKind.ShiftLeft) src1 src2))
-
-;; Helper for creating logical shift-right instructions.
-(decl x64_shr (Type Gpr Imm8Gpr) Gpr)
-(rule (x64_shr ty src1 src2)
-      (shift_r ty (ShiftKind.ShiftRightLogical) src1 src2))
-
-;; Helper for creating arithmetic shift-right instructions.
-(decl x64_sar (Type Gpr Imm8Gpr) Gpr)
-(rule (x64_sar ty src1 src2)
-      (shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2))
-
-;; Helper for creating byteswap instructions.
-;; In x64, 32- and 64-bit registers use BSWAP instruction, and
-;; for 16-bit registers one must instead use xchg or rol/ror
-(decl x64_bswap (Type Gpr) Gpr)
-(rule (x64_bswap ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.Bswap size src dst))))
-        dst))
-
-;; Helper for creating `MInst.CmpRmiR` instructions.
-(decl cmp_rmi_r (OperandSize CmpOpcode GprMemImm Gpr) ProducesFlags)
-(rule (cmp_rmi_r size opcode src1 src2)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.CmpRmiR size
-                      opcode
-                      src1
-                      src2)))
-
-;; Helper for creating `cmp` instructions.
-(decl x64_cmp (OperandSize GprMemImm Gpr) ProducesFlags)
-(rule (x64_cmp size src1 src2)
-      (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
-
-;; Helper for creating `cmp` instructions with an immediate.
-(decl x64_cmp_imm (OperandSize u32 Gpr) ProducesFlags)
-(rule (x64_cmp_imm size src1 src2)
-      (cmp_rmi_r size (CmpOpcode.Cmp) (RegMemImm.Imm src1) src2))
-
-;; Helper for creating `MInst.XmmCmpRmR` instructions.
-(decl xmm_cmp_rm_r (SseOpcode XmmMem Xmm) ProducesFlags)
-(rule (xmm_cmp_rm_r opcode src1 src2)
-      (ProducesFlags.ProducesFlagsSideEffect
-       (MInst.XmmCmpRmR opcode src1 src2)))
-
-;; Helper for creating floating-point comparison instructions (`UCOMIS[S|D]`).
-(decl x64_ucomis (Value Value) ProducesFlags)
-(rule (x64_ucomis src1 @ (value_type $F32) src2)
-      ;; N.B.: cmp can be generated more than once, so cannot do a
-      ;; load-op merge. So `put_in_xmm` for src1, not `put_in_xmm_mem`.
-      (xmm_cmp_rm_r (SseOpcode.Ucomiss) (put_in_xmm src1) (put_in_xmm src2)))
-(rule (x64_ucomis src1 @ (value_type $F64) src2)
-      (xmm_cmp_rm_r (SseOpcode.Ucomisd) (put_in_xmm src1) (put_in_xmm src2)))
-
-;; Helper for creating `test` instructions.
-(decl x64_test (OperandSize GprMemImm Gpr) ProducesFlags)
-(rule (x64_test size src1 src2)
-      (cmp_rmi_r size (CmpOpcode.Test) src1 src2))
-
-;; Helper for creating `ptest` instructions.
-(decl x64_ptest (XmmMem Xmm) ProducesFlags)
-(rule (x64_ptest src1 src2)
-      (xmm_cmp_rm_r (SseOpcode.Ptest) src1 src2))
-
-;; Helper for creating `cmove` instructions. Note that these instructions do not
-;; always result in a single emitted x86 instruction; e.g., XmmCmove uses jumps
-;; to conditionally move the selected value into an XMM register.
-(decl cmove (Type CC GprMem Gpr) ConsumesFlags)
-(rule (cmove ty cc consequent alternative)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.Cmove size cc consequent alternative dst)
-         dst)))
-
-(decl cmove_xmm (Type CC XmmMem Xmm) ConsumesFlags)
-(rule (cmove_xmm ty cc consequent alternative)
-      (let ((dst WritableXmm (temp_writable_xmm)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.XmmCmove ty cc consequent alternative dst)
-         dst)))
-
-;; Helper for creating `cmove` instructions directly from values. This allows us
-;; to special-case the `I128` types and default to the `cmove` helper otherwise.
-;; It also eliminates some `put_in_reg*` boilerplate in the lowering ISLE code.
-(decl cmove_from_values (Type CC Value Value) ConsumesFlags)
-(rule (cmove_from_values (is_multi_register_gpr_type $I128) cc consequent alternative)
-      (let ((cons ValueRegs consequent)
-            (alt ValueRegs alternative)
-            (dst1 WritableGpr (temp_writable_gpr))
-            (dst2 WritableGpr (temp_writable_gpr))
-            (size OperandSize (OperandSize.Size64))
-            (lower_cmove MInst (MInst.Cmove
-                                size cc
-                                (value_regs_get_gpr cons 0)
-                                (value_regs_get_gpr alt 0)
-                                dst1))
-            (upper_cmove MInst (MInst.Cmove
-                                size cc
-                                (value_regs_get_gpr cons 1)
-                                (value_regs_get_gpr alt 1)
-                                dst2)))
-        (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
-         lower_cmove
-         upper_cmove
-         (value_regs dst1 dst2))))
-
-(rule (cmove_from_values (is_single_register_gpr_type ty) cc consequent alternative)
-      (cmove ty cc consequent alternative))
-
-(rule (cmove_from_values (is_xmm_type ty) cc consequent alternative)
-      (cmove_xmm ty cc consequent alternative))
-
-;; Helper for creating `cmove` instructions with the logical OR of multiple
-;; flags. Note that these instructions will always result in more than one
-;; emitted x86 instruction.
-(decl cmove_or (Type CC CC GprMem Gpr) ConsumesFlags)
-(rule (cmove_or ty cc1 cc2 consequent alternative)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (tmp WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (cmove1 MInst (MInst.Cmove size cc1 consequent alternative tmp))
-            (cmove2 MInst (MInst.Cmove size cc2 consequent tmp dst)))
-        (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
-         cmove1
-         cmove2
-         dst)))
-
-(decl cmove_or_xmm (Type CC CC XmmMem Xmm) ConsumesFlags)
-(rule (cmove_or_xmm ty cc1 cc2 consequent alternative)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (tmp WritableXmm (temp_writable_xmm))
-            (cmove1 MInst (MInst.XmmCmove ty cc1 consequent alternative tmp))
-            (cmove2 MInst (MInst.XmmCmove ty cc2 consequent tmp dst)))
-        (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
-         cmove1
-         cmove2
-         dst)))
-
-;; Helper for creating `cmove_or` instructions directly from values. This allows
-;; us to special-case the `I128` types and default to the `cmove_or` helper
-;; otherwise.
-(decl cmove_or_from_values (Type CC CC Value Value) ConsumesFlags)
-(rule (cmove_or_from_values (is_multi_register_gpr_type $I128) cc1 cc2 consequent alternative)
-      (let ((cons ValueRegs consequent)
-            (alt ValueRegs alternative)
-            (dst1 WritableGpr (temp_writable_gpr))
-            (dst2 WritableGpr (temp_writable_gpr))
-            (tmp1 WritableGpr (temp_writable_gpr))
-            (tmp2 WritableGpr (temp_writable_gpr))
-            (size OperandSize (OperandSize.Size64))
-            (cmove1 MInst (MInst.Cmove size cc1 (value_regs_get_gpr cons 0) (value_regs_get_gpr alt 0) tmp1))
-            (cmove2 MInst (MInst.Cmove size cc2 (value_regs_get_gpr cons 0) tmp1 dst1))
-            (cmove3 MInst (MInst.Cmove size cc1 (value_regs_get_gpr cons 1) (value_regs_get_gpr alt 1) tmp2))
-            (cmove4 MInst (MInst.Cmove size cc2 (value_regs_get_gpr cons 1) tmp2 dst2)))
-        (ConsumesFlags.ConsumesFlagsFourTimesReturnsValueRegs
-         cmove1
-         cmove2
-         cmove3
-         cmove4
-         (value_regs dst1 dst2))))
-
-(rule (cmove_or_from_values (is_single_register_gpr_type ty) cc1 cc2 consequent alternative)
-      (cmove_or ty cc1 cc2 consequent alternative))
-
-(rule (cmove_or_from_values (is_xmm_type ty) cc1 cc2 consequent alternative)
-      (cmove_or_xmm ty cc1 cc2 consequent alternative))
-
-;; Helper for creating `MInst.Setcc` instructions.
-(decl x64_setcc (CC) ConsumesFlags)
-(rule (x64_setcc cc)
-      (let ((dst WritableGpr (temp_writable_gpr)))
-        (ConsumesFlags.ConsumesFlagsReturnsReg
-         (MInst.Setcc cc dst)
-         dst)))
-
-;; Helper for creating `MInst.XmmRmR` instructions.
-(decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm)
-(rule (xmm_rm_r ty op src1 src2)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR op src1 src2 dst))))
-        dst))
-
-;; Helper for creating `paddb` instructions.
-(decl x64_paddb (Xmm XmmMem) Xmm)
-(rule (x64_paddb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2))
-
-;; Helper for creating `paddw` instructions.
-(decl x64_paddw (Xmm XmmMem) Xmm)
-(rule (x64_paddw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2))
-
-;; Helper for creating `paddd` instructions.
-(decl x64_paddd (Xmm XmmMem) Xmm)
-(rule (x64_paddd src1 src2)
-      (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2))
-
-;; Helper for creating `paddq` instructions.
-(decl x64_paddq (Xmm XmmMem) Xmm)
-(rule (x64_paddq src1 src2)
-      (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2))
-
-;; Helper for creating `paddsb` instructions.
-(decl x64_paddsb (Xmm XmmMem) Xmm)
-(rule (x64_paddsb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2))
-
-;; Helper for creating `paddsw` instructions.
-(decl x64_paddsw (Xmm XmmMem) Xmm)
-(rule (x64_paddsw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2))
-
-;; Helper for creating `paddusb` instructions.
-(decl x64_paddusb (Xmm XmmMem) Xmm)
-(rule (x64_paddusb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2))
-
-;; Helper for creating `paddusw` instructions.
-(decl x64_paddusw (Xmm XmmMem) Xmm)
-(rule (x64_paddusw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2))
-
-;; Helper for creating `psubb` instructions.
-(decl x64_psubb (Xmm XmmMem) Xmm)
-(rule (x64_psubb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2))
-
-;; Helper for creating `psubw` instructions.
-(decl x64_psubw (Xmm XmmMem) Xmm)
-(rule (x64_psubw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2))
-
-;; Helper for creating `psubd` instructions.
-(decl x64_psubd (Xmm XmmMem) Xmm)
-(rule (x64_psubd src1 src2)
-      (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2))
-
-;; Helper for creating `psubq` instructions.
-(decl x64_psubq (Xmm XmmMem) Xmm)
-(rule (x64_psubq src1 src2)
-      (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2))
-
-;; Helper for creating `psubsb` instructions.
-(decl x64_psubsb (Xmm XmmMem) Xmm)
-(rule (x64_psubsb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2))
-
-;; Helper for creating `psubsw` instructions.
-(decl x64_psubsw (Xmm XmmMem) Xmm)
-(rule (x64_psubsw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2))
-
-;; Helper for creating `psubusb` instructions.
-(decl x64_psubusb (Xmm XmmMem) Xmm)
-(rule (x64_psubusb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2))
-
-;; Helper for creating `psubusw` instructions.
-(decl x64_psubusw (Xmm XmmMem) Xmm)
-(rule (x64_psubusw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2))
-
-;; Helper for creating `pavgb` instructions.
-(decl x64_pavgb (Xmm XmmMem) Xmm)
-(rule (x64_pavgb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2))
-
-;; Helper for creating `pavgw` instructions.
-(decl x64_pavgw (Xmm XmmMem) Xmm)
-(rule (x64_pavgw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2))
-
-;; Helper for creating `pand` instructions.
-(decl x64_pand (Xmm XmmMem) Xmm)
-(rule (x64_pand src1 src2)
-      (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2))
-
-;; Helper for creating `andps` instructions.
-(decl x64_andps (Xmm XmmMem) Xmm)
-(rule (x64_andps src1 src2)
-      (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2))
-
-;; Helper for creating `andpd` instructions.
-(decl x64_andpd (Xmm XmmMem) Xmm)
-(rule (x64_andpd src1 src2)
-      (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2))
-
-;; Helper for creating `por` instructions.
-(decl x64_por (Xmm XmmMem) Xmm)
-(rule (x64_por src1 src2)
-      (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2))
-
-;; Helper for creating `orps` instructions.
-(decl x64_orps (Xmm XmmMem) Xmm)
-(rule (x64_orps src1 src2)
-      (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2))
-
-;; Helper for creating `orpd` instructions.
-(decl x64_orpd (Xmm XmmMem) Xmm)
-(rule (x64_orpd src1 src2)
-      (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2))
-
-;; Helper for creating `pxor` instructions.
-(decl x64_pxor (Xmm XmmMem) Xmm)
-(rule (x64_pxor src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2))
-
-;; Helper for creating `xorps` instructions.
-(decl x64_xorps (Xmm XmmMem) Xmm)
-(rule (x64_xorps src1 src2)
-      (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2))
-
-;; Helper for creating `xorpd` instructions.
-(decl x64_xorpd (Xmm XmmMem) Xmm)
-(rule (x64_xorpd src1 src2)
-      (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2))
-
-;; Helper for creating `pmullw` instructions.
-(decl x64_pmullw (Xmm XmmMem) Xmm)
-(rule (x64_pmullw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2))
-
-;; Helper for creating `pmulld` instructions.
-(decl x64_pmulld (Xmm XmmMem) Xmm)
-(rule (x64_pmulld src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2))
-
-;; Helper for creating `pmulhw` instructions.
-(decl x64_pmulhw (Xmm XmmMem) Xmm)
-(rule (x64_pmulhw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2))
-
-;; Helper for creating `pmulhrsw` instructions.
-(decl x64_pmulhrsw (Xmm XmmMem) Xmm)
-(rule (x64_pmulhrsw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pmulhrsw) src1 src2))
-
-;; Helper for creating `pmulhuw` instructions.
-(decl x64_pmulhuw (Xmm XmmMem) Xmm)
-(rule (x64_pmulhuw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2))
-
-;; Helper for creating `pmuldq` instructions.
-(decl x64_pmuldq (Xmm XmmMem) Xmm)
-(rule (x64_pmuldq src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2))
-
-;; Helper for creating `pmuludq` instructions.
-(decl x64_pmuludq (Xmm XmmMem) Xmm)
-(rule (x64_pmuludq src1 src2)
-      (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2))
-
-;; Helper for creating `punpckhwd` instructions.
-(decl x64_punpckhwd (Xmm XmmMem) Xmm)
-(rule (x64_punpckhwd src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2))
-
-;; Helper for creating `punpcklwd` instructions.
-(decl x64_punpcklwd (Xmm XmmMem) Xmm)
-(rule (x64_punpcklwd src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2))
-
-;; Helper for creating `unpcklps` instructions.
-(decl x64_unpcklps (Xmm XmmMem) Xmm)
-(rule (x64_unpcklps src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Unpcklps) src1 src2))
-
-;; Helper for creating `andnps` instructions.
-(decl x64_andnps (Xmm XmmMem) Xmm)
-(rule (x64_andnps src1 src2)
-      (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2))
-
-;; Helper for creating `andnpd` instructions.
-(decl x64_andnpd (Xmm XmmMem) Xmm)
-(rule (x64_andnpd src1 src2)
-      (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2))
-
-;; Helper for creating `pandn` instructions.
-(decl x64_pandn (Xmm XmmMem) Xmm)
-(rule (x64_pandn src1 src2)
-      (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2))
-
-;; Helper for creating `addss` instructions.
-(decl x64_addss (Xmm XmmMem) Xmm)
-(rule (x64_addss src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Addss) src1 src2))
-
-;; Helper for creating `addsd` instructions.
-(decl x64_addsd (Xmm XmmMem) Xmm)
-(rule (x64_addsd src1 src2)
-      (xmm_rm_r $F64 (SseOpcode.Addsd) src1 src2))
-
-;; Helper for creating `addps` instructions.
-(decl x64_addps (Xmm XmmMem) Xmm)
-(rule (x64_addps src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Addps) src1 src2))
-
-;; Helper for creating `addpd` instructions.
-(decl x64_addpd (Xmm XmmMem) Xmm)
-(rule (x64_addpd src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Addpd) src1 src2))
-
-;; Helper for creating `subss` instructions.
-(decl x64_subss (Xmm XmmMem) Xmm)
-(rule (x64_subss src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Subss) src1 src2))
-
-;; Helper for creating `subsd` instructions.
-(decl x64_subsd (Xmm XmmMem) Xmm)
-(rule (x64_subsd src1 src2)
-      (xmm_rm_r $F64 (SseOpcode.Subsd) src1 src2))
-
-;; Helper for creating `subps` instructions.
-(decl x64_subps (Xmm XmmMem) Xmm)
-(rule (x64_subps src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Subps) src1 src2))
-
-;; Helper for creating `subpd` instructions.
-(decl x64_subpd (Xmm XmmMem) Xmm)
-(rule (x64_subpd src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Subpd) src1 src2))
-
-;; Helper for creating `mulss` instructions.
-(decl x64_mulss (Xmm XmmMem) Xmm)
-(rule (x64_mulss src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Mulss) src1 src2))
-
-;; Helper for creating `mulsd` instructions.
-(decl x64_mulsd (Xmm XmmMem) Xmm)
-(rule (x64_mulsd src1 src2)
-      (xmm_rm_r $F64 (SseOpcode.Mulsd) src1 src2))
-
-;; Helper for creating `mulps` instructions.
-(decl x64_mulps (Xmm XmmMem) Xmm)
-(rule (x64_mulps src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Mulps) src1 src2))
-
-;; Helper for creating `mulpd` instructions.
-(decl x64_mulpd (Xmm XmmMem) Xmm)
-(rule (x64_mulpd src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Mulpd) src1 src2))
-
-;; Helper for creating `divss` instructions.
-(decl x64_divss (Xmm XmmMem) Xmm)
-(rule (x64_divss src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Divss) src1 src2))
-
-;; Helper for creating `divsd` instructions.
-(decl x64_divsd (Xmm XmmMem) Xmm)
-(rule (x64_divsd src1 src2)
-      (xmm_rm_r $F64 (SseOpcode.Divsd) src1 src2))
-
-;; Helper for creating `divps` instructions.
-(decl x64_divps (Xmm XmmMem) Xmm)
-(rule (x64_divps src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Divps) src1 src2))
-
-;; Helper for creating `divpd` instructions.
-(decl x64_divpd (Xmm XmmMem) Xmm)
-(rule (x64_divpd src1 src2)
-      (xmm_rm_r $F32 (SseOpcode.Divpd) src1 src2))
-
-(decl sse_blend_op (Type) SseOpcode)
-(rule 1 (sse_blend_op $F32X4) (SseOpcode.Blendvps))
-(rule 1 (sse_blend_op $F64X2) (SseOpcode.Blendvpd))
-
-;; Priority 0 because multi_lane overlaps with the previous two type patterns.
-(rule 0 (sse_blend_op (multi_lane _bits _lanes)) (SseOpcode.Pblendvb))
-
-(decl sse_mov_op (Type) SseOpcode)
-(rule 1 (sse_mov_op $F32X4) (SseOpcode.Movaps))
-(rule 1 (sse_mov_op $F64X2) (SseOpcode.Movapd))
-
-;; Priority 0 because multi_lane overlaps with the previous two type patterns.
-(rule 0 (sse_mov_op (multi_lane _bits _lanes)) (SseOpcode.Movdqa))
-
-(decl xmm_rm_r_blend (SseOpcode Xmm XmmMem Xmm) Xmm)
-(rule (xmm_rm_r_blend op src1 src2 mask)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmRBlend op src1 src2 mask dst))))
-        dst))
-
-;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions.
-(decl x64_blend (Type Xmm XmmMem Xmm) Xmm)
-(rule (x64_blend ty mask src1 src2)
-      (xmm_rm_r_blend (sse_blend_op ty) src2 src1 mask))
-
-;; Helper for creating `blendvpd` instructions.
-(decl x64_blendvpd (Xmm XmmMem Xmm) Xmm)
-(rule (x64_blendvpd src1 src2 mask)
-      (xmm_rm_r_blend (SseOpcode.Blendvpd) src1 src2 mask))
-
-;; Helper for creating `movsd` instructions.
-(decl x64_movsd_regmove (Xmm XmmMem) Xmm)
-(rule (x64_movsd_regmove src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Movsd) src1 src2))
-
-;; Helper for creating `movlhps` instructions.
-(decl x64_movlhps (Xmm XmmMem) Xmm)
-(rule (x64_movlhps src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Movlhps) src1 src2))
-
-;; Helpers for creating `pmaxs*` instructions.
-(decl x64_pmaxs (Type Xmm XmmMem) Xmm)
-(rule (x64_pmaxs $I8X16 x y) (x64_pmaxsb x y))
-(rule (x64_pmaxs $I16X8 x y) (x64_pmaxsw x y))
-(rule (x64_pmaxs $I32X4 x y) (x64_pmaxsd x y))
-;; No $I64X2 version (PMAXSQ) in SSE4.1.
-(decl x64_pmaxsb (Xmm XmmMem) Xmm)
-(rule (x64_pmaxsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsb) src1 src2))
-(decl x64_pmaxsw (Xmm XmmMem) Xmm)
-(rule (x64_pmaxsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsw) src1 src2))
-(decl x64_pmaxsd (Xmm XmmMem) Xmm)
-(rule (x64_pmaxsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsd) src1 src2))
-
-;; Helpers for creating `pmins*` instructions.
-(decl x64_pmins (Type Xmm XmmMem) Xmm)
-(rule (x64_pmins $I8X16 x y) (x64_pminsb x y))
-(rule (x64_pmins $I16X8 x y) (x64_pminsw x y))
-(rule (x64_pmins $I32X4 x y) (x64_pminsd x y))
-;; No $I64X2 version (PMINSQ) in SSE4.1.
-(decl x64_pminsb (Xmm XmmMem) Xmm)
-(rule (x64_pminsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsb) src1 src2))
-(decl x64_pminsw (Xmm XmmMem) Xmm)
-(rule (x64_pminsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pminsw) src1 src2))
-(decl x64_pminsd (Xmm XmmMem) Xmm)
-(rule (x64_pminsd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Pminsd) src1 src2))
-
-;; Helpers for creating `pmaxu*` instructions.
-(decl x64_pmaxu (Type Xmm XmmMem) Xmm)
-(rule (x64_pmaxu $I8X16 x y) (x64_pmaxub x y))
-(rule (x64_pmaxu $I16X8 x y) (x64_pmaxuw x y))
-(rule (x64_pmaxu $I32X4 x y) (x64_pmaxud x y))
-;; No $I64X2 version (PMAXUQ) in SSE4.1.
-(decl x64_pmaxub (Xmm XmmMem) Xmm)
-(rule (x64_pmaxub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxub) src1 src2))
-(decl x64_pmaxuw (Xmm XmmMem) Xmm)
-(rule (x64_pmaxuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxuw) src1 src2))
-(decl x64_pmaxud (Xmm XmmMem) Xmm)
-(rule (x64_pmaxud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxud) src1 src2))
-
-;; Helper for creating `pminu*` instructions.
-(decl x64_pminu (Type Xmm XmmMem) Xmm)
-(rule (x64_pminu $I8X16 x y) (x64_pminub x y))
-(rule (x64_pminu $I16X8 x y) (x64_pminuw x y))
-(rule (x64_pminu $I32X4 x y) (x64_pminud x y))
-;; No $I64X2 version (PMINUQ) in SSE4.1.
-(decl x64_pminub (Xmm XmmMem) Xmm)
-(rule (x64_pminub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminub) src1 src2))
-(decl x64_pminuw (Xmm XmmMem) Xmm)
-(rule (x64_pminuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminuw) src1 src2))
-(decl x64_pminud (Xmm XmmMem) Xmm)
-(rule (x64_pminud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2))
-
-;; Helper for creating `punpcklbw` instructions.
-(decl x64_punpcklbw (Xmm XmmMem) Xmm)
-(rule (x64_punpcklbw src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2))
-
-;; Helper for creating `punpckhbw` instructions.
-(decl x64_punpckhbw (Xmm XmmMem) Xmm)
-(rule (x64_punpckhbw src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2))
-
-;; Helper for creating `packsswb` instructions.
-(decl x64_packsswb (Xmm XmmMem) Xmm)
-(rule (x64_packsswb src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2))
-
-;; Helper for creating `packssdw` instructions.
-(decl x64_packssdw (Xmm XmmMem) Xmm)
-(rule (x64_packssdw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Packssdw) src1 src2))
-
-;; Helper for creating `packuswb` instructions.
-(decl x64_packuswb (Xmm XmmMem) Xmm)
-(rule (x64_packuswb src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Packuswb) src1 src2))
-
-;; Helper for creating `packusdw` instructions.
-(decl x64_packusdw (Xmm XmmMem) Xmm)
-(rule (x64_packusdw src1 src2)
-      (xmm_rm_r $I16X8 (SseOpcode.Packusdw) src1 src2))
-
-;; Helper for creating `MInst.XmmRmRImm` instructions.
-(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Xmm)
-(rule (xmm_rm_r_imm op src1 src2 imm size)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmRImm op
-                                           src1
-                                           src2
-                                           dst
-                                           imm
-                                           size))))
-        dst))
-
-;; Helper for creating `palignr` instructions.
-(decl x64_palignr (Xmm XmmMem u8 OperandSize) Xmm)
-(rule (x64_palignr src1 src2 imm size)
-      (xmm_rm_r_imm (SseOpcode.Palignr)
-                    src1
-                    src2
-                    imm
-                    size))
-
-;; Helpers for creating `cmpp*` instructions.
-(decl x64_cmpp (Type Xmm XmmMem FcmpImm) Xmm)
-(rule (x64_cmpp $F32X4 x y imm) (x64_cmpps x y imm))
-(rule (x64_cmpp $F64X2 x y imm) (x64_cmppd x y imm))
-
-(decl x64_cmpps (Xmm XmmMem FcmpImm) Xmm)
-(rule (x64_cmpps src1 src2 imm)
-      (xmm_rm_r_imm (SseOpcode.Cmpps)
-                    src1
-                    src2
-                    (encode_fcmp_imm imm)
-                    (OperandSize.Size32)))
-
-;; Note that `Size32` is intentional despite this being used for 64-bit
-;; operations, since this presumably induces the correct encoding of the
-;; instruction.
-(decl x64_cmppd (Xmm XmmMem FcmpImm) Xmm)
-(rule (x64_cmppd src1 src2 imm)
-      (xmm_rm_r_imm (SseOpcode.Cmppd)
-                    src1
-                    src2
-                    (encode_fcmp_imm imm)
-                    (OperandSize.Size32)))
-
-;; Helper for creating `pinsrb` instructions.
-(decl x64_pinsrb (Xmm GprMem u8) Xmm)
-(rule (x64_pinsrb src1 src2 lane)
-      (xmm_rm_r_imm (SseOpcode.Pinsrb)
-                    src1
-                    src2
-                    lane
-                    (OperandSize.Size32)))
-
-;; Helper for creating `pinsrw` instructions.
-(decl x64_pinsrw (Xmm GprMem u8) Xmm)
-(rule (x64_pinsrw src1 src2 lane)
-      (xmm_rm_r_imm (SseOpcode.Pinsrw)
-                    src1
-                    src2
-                    lane
-                    (OperandSize.Size32)))
-
-;; Helper for creating `pinsrd` instructions.
-(decl x64_pinsrd (Xmm GprMem u8 OperandSize) Xmm)
-(rule (x64_pinsrd src1 src2 lane size)
-      (xmm_rm_r_imm (SseOpcode.Pinsrd)
-                    src1
-                    src2
-                    lane
-                    size))
-
-;; Helper for constructing `XmmUnaryRmRImm` instructions.
-(decl xmm_unary_rm_r_imm (SseOpcode XmmMem u8) Xmm)
-(rule (xmm_unary_rm_r_imm op src1 imm)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmRImm op src1 imm dst))))
-        dst))
-
-;; Helper for creating `roundss` instructions.
-(decl x64_roundss (XmmMem RoundImm) Xmm)
-(rule (x64_roundss src1 round)
-      (xmm_unary_rm_r_imm (SseOpcode.Roundss) src1 (encode_round_imm round)))
-
-;; Helper for creating `roundsd` instructions.
-(decl x64_roundsd (XmmMem RoundImm) Xmm)
-(rule (x64_roundsd src1 round)
-      (xmm_unary_rm_r_imm (SseOpcode.Roundsd) src1 (encode_round_imm round)))
-
-;; Helper for creating `roundps` instructions.
-(decl x64_roundps (XmmMem RoundImm) Xmm)
-(rule (x64_roundps src1 round)
-      (xmm_unary_rm_r_imm (SseOpcode.Roundps) src1 (encode_round_imm round)))
-
-;; Helper for creating `roundpd` instructions.
-(decl x64_roundpd (XmmMem RoundImm) Xmm)
-(rule (x64_roundpd src1 round)
-      (xmm_unary_rm_r_imm (SseOpcode.Roundpd) src1 (encode_round_imm round)))
-
-;; Helper for creating `pmaddwd` instructions.
-(decl x64_pmaddwd (Xmm XmmMem) Xmm)
-(rule (x64_pmaddwd src1 src2)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Pmaddwd)
-                                        src1
-                                        src2
-                                        dst))))
-        dst))
-
-(decl x64_pmaddubsw (Xmm XmmMem) Xmm)
-(rule (x64_pmaddubsw src1 src2)
-      (xmm_rm_r $I8X16 (SseOpcode.Pmaddubsw) src1 src2))
-
-;; Helper for creating `insertps` instructions.
-(decl x64_insertps (Xmm XmmMem u8) Xmm)
-(rule (x64_insertps src1 src2 lane)
-      (xmm_rm_r_imm (SseOpcode.Insertps)
-                    src1
-                    src2
-                    lane
-                    (OperandSize.Size32)))
-
-;; Helper for creating `pshufd` instructions.
-(decl x64_pshufd (XmmMem u8 OperandSize) Xmm)
-(rule (x64_pshufd src imm size)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd)
-                                           dst
-                                           src
-                                           dst
-                                           imm
-                                           size))))
-        dst))
-
-;; Helper for creating `pshufb` instructions.
-(decl x64_pshufb (Xmm XmmMem) Xmm)
-(rule (x64_pshufb src1 src2)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Pshufb)
-                                        src1
-                                        src2
-                                        dst))))
-        dst))
-
-;; Helper for creating `shufps` instructions.
-(decl x64_shufps (Xmm XmmMem u8) Xmm)
-(rule (x64_shufps src1 src2 byte)
-      (xmm_rm_r_imm (SseOpcode.Shufps)
-                    src1
-                    src2
-                    byte
-                    (OperandSize.Size32)))
-
-;; Helper for creating `MInst.XmmUnaryRmR` instructions.
-(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm)
-(rule (xmm_unary_rm_r op src)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR op src dst))))
-        dst))
-
-;; Helper for creating `pabsb` instructions.
-(decl x64_pabsb (XmmMem) Xmm)
-(rule (x64_pabsb src)
-      (xmm_unary_rm_r (SseOpcode.Pabsb) src))
-
-;; Helper for creating `pabsw` instructions.
-(decl x64_pabsw (XmmMem) Xmm)
-(rule (x64_pabsw src)
-      (xmm_unary_rm_r (SseOpcode.Pabsw) src))
-
-;; Helper for creating `pabsd` instructions.
-(decl x64_pabsd (XmmMem) Xmm)
-(rule (x64_pabsd src)
-      (xmm_unary_rm_r (SseOpcode.Pabsd) src))
-
-;; Helper for creating `MInst.XmmUnaryRmREvex` instructions.
-(decl xmm_unary_rm_r_evex (Avx512Opcode XmmMem) Xmm)
-(rule (xmm_unary_rm_r_evex op src)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmREvex op src dst))))
-        dst))
-
-;; Helper for creating `vcvtudq2ps` instructions.
-(decl x64_vcvtudq2ps (XmmMem) Xmm)
-(rule (x64_vcvtudq2ps src)
-      (xmm_unary_rm_r_evex (Avx512Opcode.Vcvtudq2ps) src))
-
-;; Helper for creating `vpabsq` instructions.
-(decl x64_vpabsq (XmmMem) Xmm)
-(rule (x64_vpabsq src)
-      (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src))
-
-;; Helper for creating `vpopcntb` instructions.
-(decl x64_vpopcntb (XmmMem) Xmm)
-(rule (x64_vpopcntb src)
-      (xmm_unary_rm_r_evex (Avx512Opcode.Vpopcntb) src))
-
-;; Helper for creating `MInst.XmmRmREvex` instructions.
-(decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm)
-(rule (xmm_rm_r_evex op src1 src2)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmREvex op
-                                            src1
-                                            src2
-                                            dst))))
-        dst))
-
-;; Helper for creating `vpmullq` instructions.
-;;
-;; Requires AVX-512 vl and dq.
-(decl x64_vpmullq (XmmMem Xmm) Xmm)
-(rule (x64_vpmullq src1 src2)
-      (xmm_rm_r_evex (Avx512Opcode.Vpmullq)
-                     src1
-                     src2))
-
-;; Helper for creating `vpermi2b` instructions.
-;;
-;; Requires AVX-512 vl and vbmi extensions.
-(decl x64_vpermi2b (Xmm Xmm Xmm) Xmm)
-(rule (x64_vpermi2b src1 src2 src3)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmREvex3 (Avx512Opcode.Vpermi2b)
-                                             src1
-                                             src2
-                                             src3
-                                             dst))))
-        dst))
-
-;; Helper for creating `MInst.MulHi` instructions.
-;;
-;; Returns the (lo, hi) register halves of the multiplication.
-(decl mul_hi (Type bool Gpr GprMem) ValueRegs)
-(rule (mul_hi ty signed src1 src2)
-      (let ((dst_lo WritableGpr (temp_writable_gpr))
-            (dst_hi WritableGpr (temp_writable_gpr))
-            (size OperandSize (raw_operand_size_of_type ty))
-            (_ Unit (emit (MInst.MulHi size
-                                       signed
-                                       src1
-                                       src2
-                                       dst_lo
-                                       dst_hi))))
-        (value_gprs dst_lo dst_hi)))
-
-;; Helper for creating `mul` instructions that return both the lower and
-;; (unsigned) higher halves of the result.
-(decl mulhi_u (Type Gpr GprMem) ValueRegs)
-(rule (mulhi_u ty src1 src2)
-      (mul_hi ty $false src1 src2))
-
-;; Helper for creating `MInst.XmmRmiXmm` instructions.
-(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemImm) Xmm)
-(rule (xmm_rmi_xmm op src1 src2)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmiReg op
-                                           src1
-                                           src2
-                                           dst))))
-        dst))
-
-;; Helper for creating `psllw` instructions.
-(decl x64_psllw (Xmm XmmMemImm) Xmm)
-(rule (x64_psllw src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psllw) src1 src2))
-
-;; Helper for creating `pslld` instructions.
-(decl x64_pslld (Xmm XmmMemImm) Xmm)
-(rule (x64_pslld src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Pslld) src1 src2))
-
-;; Helper for creating `psllq` instructions.
-(decl x64_psllq (Xmm XmmMemImm) Xmm)
-(rule (x64_psllq src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psllq) src1 src2))
-
-;; Helper for creating `psrlw` instructions.
-(decl x64_psrlw (Xmm XmmMemImm) Xmm)
-(rule (x64_psrlw src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psrlw) src1 src2))
-
-;; Helper for creating `psrld` instructions.
-(decl x64_psrld (Xmm XmmMemImm) Xmm)
-(rule (x64_psrld src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psrld) src1 src2))
-
-;; Helper for creating `psrlq` instructions.
-(decl x64_psrlq (Xmm XmmMemImm) Xmm)
-(rule (x64_psrlq src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psrlq) src1 src2))
-
-;; Helper for creating `psraw` instructions.
-(decl x64_psraw (Xmm XmmMemImm) Xmm)
-(rule (x64_psraw src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psraw) src1 src2))
-
-;; Helper for creating `psrad` instructions.
-(decl x64_psrad (Xmm XmmMemImm) Xmm)
-(rule (x64_psrad src1 src2)
-      (xmm_rmi_xmm (SseOpcode.Psrad) src1 src2))
-
-;; Helper for creating `pextrb` instructions.
-(decl x64_pextrb (Type Xmm u8) Gpr)
-(rule (x64_pextrb ty src lane)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrb)
-                                           dst
-                                           src
-                                           dst
-                                           lane
-                                           (operand_size_of_type_32_64 (lane_type ty))))))
-        dst))
-
-;; Helper for creating `pextrw` instructions.
-(decl x64_pextrw (Type Xmm u8) Gpr)
-(rule (x64_pextrw ty src lane)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrw)
-                                           dst
-                                           src
-                                           dst
-                                           lane
-                                           (operand_size_of_type_32_64 (lane_type ty))))))
-        dst))
-
-;; Helper for creating `pextrd` instructions.
-(decl x64_pextrd (Type Xmm u8) Gpr)
-(rule (x64_pextrd ty src lane)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd)
-                                           dst
-                                           src
-                                           dst
-                                           lane
-                                           (operand_size_of_type_32_64 (lane_type ty))))))
-        dst))
-
-;; Helper for creating `MInst.XmmToGpr` instructions.
-(decl xmm_to_gpr (SseOpcode Xmm OperandSize) Gpr)
-(rule (xmm_to_gpr op src size)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmToGpr op src dst size))))
-        dst))
-
-;; Helper for creating `pmovmskb` instructions.
-(decl x64_pmovmskb (OperandSize Xmm) Gpr)
-(rule (x64_pmovmskb size src)
-      (xmm_to_gpr (SseOpcode.Pmovmskb) src size))
-
-;; Helper for creating `movmskps` instructions.
-(decl x64_movmskps (OperandSize Xmm) Gpr)
-(rule (x64_movmskps size src)
-      (xmm_to_gpr (SseOpcode.Movmskps) src size))
-
-;; Helper for creating `movmskpd` instructions.
-(decl x64_movmskpd (OperandSize Xmm) Gpr)
-(rule (x64_movmskpd size src)
-      (xmm_to_gpr (SseOpcode.Movmskpd) src size))
-
-;; Helper for creating `MInst.GprToXmm` instructions.
-(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm)
-(rule (gpr_to_xmm op src size)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.GprToXmm op src dst size))))
-        dst))
-
-;; Helper for creating `not` instructions.
-(decl x64_not (Type Gpr) Gpr)
-(rule (x64_not ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.Not size src dst))))
-        dst))
-
-;; Helper for creating `neg` instructions.
-(decl x64_neg (Type Gpr) Gpr)
-(rule (x64_neg ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (raw_operand_size_of_type ty))
-            (_ Unit (emit (MInst.Neg size src dst))))
-        dst))
-
-;; Helper for creating `neg` instructions whose flags are also used.
-(decl x64_neg_paired (Type Gpr) ProducesFlags)
-(rule (x64_neg_paired ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (raw_operand_size_of_type ty))
-            (inst MInst (MInst.Neg size src dst)))
-        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst dst)))
-
-(decl x64_lea (SyntheticAmode) Gpr)
-(rule (x64_lea addr)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.LoadEffectiveAddress addr dst))))
-        dst))
-
-;; Helper for creating `ud2` instructions.
-(decl x64_ud2 (TrapCode) SideEffectNoResult)
-(rule (x64_ud2 code)
-      (SideEffectNoResult.Inst (MInst.Ud2 code)))
-
-;; Helper for creating `hlt` instructions.
-(decl x64_hlt () SideEffectNoResult)
-(rule (x64_hlt)
-      (SideEffectNoResult.Inst (MInst.Hlt)))
-
-;; Helper for creating `lzcnt` instructions.
-(decl x64_lzcnt (Type Gpr) Gpr)
-(rule (x64_lzcnt ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Lzcnt) src dst))))
-        dst))
-
-;; Helper for creating `tzcnt` instructions.
-(decl x64_tzcnt (Type Gpr) Gpr)
-(rule (x64_tzcnt ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Tzcnt) src dst))))
-        dst))
-
-;; Helper for creating `bsr` instructions.
-(decl x64_bsr (Type Gpr) ProducesFlags)
-(rule (x64_bsr ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (inst MInst (MInst.UnaryRmR size (UnaryRmROpcode.Bsr) src dst)))
-        (ProducesFlags.ProducesFlagsReturnsReg inst dst)))
-
-;; Helper for creating `bsr + cmov` instruction pairs that produce the
-;; result of the `bsr`, or `alt` if the input was zero.
-(decl bsr_or_else (Type Gpr Gpr) Gpr)
-(rule (bsr_or_else ty src alt)
-      (let ((bsr ProducesFlags (x64_bsr ty src))
-            ;; Manually extract the result from the bsr, then ignore
-            ;; it below, since we need to thread it into the cmove
-            ;; before we pass the cmove to with_flags_reg.
-            (bsr_result Gpr (produces_flags_get_reg bsr))
-            (cmove ConsumesFlags (cmove ty (CC.Z) alt bsr_result)))
-        (with_flags_reg (produces_flags_ignore bsr) cmove)))
-
-;; Helper for creating `bsf` instructions.
-(decl x64_bsf (Type Gpr) ProducesFlags)
-(rule (x64_bsf ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (inst MInst (MInst.UnaryRmR size (UnaryRmROpcode.Bsf) src dst)))
-        (ProducesFlags.ProducesFlagsReturnsReg inst dst)))
-
-;; Helper for creating `bsf + cmov` instruction pairs that produce the
-;; result of the `bsf`, or `alt` if the input was zero.
-(decl bsf_or_else (Type Gpr Gpr) Gpr)
-(rule (bsf_or_else ty src alt)
-      (let ((bsf ProducesFlags (x64_bsf ty src))
-            ;; Manually extract the result from the bsf, then ignore
-            ;; it below, since we need to thread it into the cmove
-            ;; before we pass the cmove to with_flags_reg.
-            (bsf_result Gpr (produces_flags_get_reg bsf))
-            (cmove ConsumesFlags (cmove ty (CC.Z) alt bsf_result)))
-        (with_flags_reg (produces_flags_ignore bsf) cmove)))
-
-;; Helper for creating `popcnt` instructions.
-(decl x64_popcnt (Type Gpr) Gpr)
-(rule (x64_popcnt ty src)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Popcnt) src dst))))
-        dst))
-
-;; Helper for creating `xmm_min_max_seq` psuedo-instructions.
-(decl xmm_min_max_seq (Type bool Xmm Xmm) Xmm)
-(rule (xmm_min_max_seq ty is_min lhs rhs)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (size OperandSize (operand_size_of_type_32_64 ty))
-            (_ Unit (emit (MInst.XmmMinMaxSeq size is_min lhs rhs dst))))
-        dst))
-
-;; Helper for creating `minss` instructions.
-(decl x64_minss (Xmm Xmm) Xmm)
-(rule (x64_minss x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minss) x y dst))))
-        dst))
-
-;; Helper for creating `minsd` instructions.
-(decl x64_minsd (Xmm Xmm) Xmm)
-(rule (x64_minsd x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minsd) x y dst))))
-        dst))
-
-
-;; Helper for creating `minps` instructions.
-(decl x64_minps (Xmm Xmm) Xmm)
-(rule (x64_minps x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minps) x y dst))))
-        dst))
-
-;; Helper for creating `minpd` instructions.
-(decl x64_minpd (Xmm Xmm) Xmm)
-(rule (x64_minpd x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minpd) x y dst))))
-        dst))
-
-;; Helper for creating `maxss` instructions.
-(decl x64_maxss (Xmm Xmm) Xmm)
-(rule (x64_maxss x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Maxss) x y dst))))
-        dst))
-
-;; Helper for creating `maxsd` instructions.
-(decl x64_maxsd (Xmm Xmm) Xmm)
-(rule (x64_maxsd x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Maxsd) x y dst))))
-        dst))
-
-;; Helper for creating `maxps` instructions.
-(decl x64_maxps (Xmm Xmm) Xmm)
-(rule (x64_maxps x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Maxps) x y dst))))
-        dst))
-
-;; Helper for creating `maxpd` instructions.
-(decl x64_maxpd (Xmm Xmm) Xmm)
-(rule (x64_maxpd x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Maxpd) x y dst))))
-        dst))
-
-
-;; Helper for creating `MInst.XmmRmRVex` instructions.
-(decl xmm_rmr_vex (AvxOpcode Xmm Xmm XmmMem) Xmm)
-(rule (xmm_rmr_vex op src1 src2 src3)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmRVex op
-                                           src1
-                                           src2
-                                           src3
-                                           dst))))
-        dst))
-
-;; Helper for creating `vfmadd213ss` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213ss (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213ss x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213ss) x y z))
-
-;; Helper for creating `vfmadd213sd` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213sd (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213sd x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213sd) x y z))
-
-;; Helper for creating `vfmadd213ps` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213ps (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213ps x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213ps) x y z))
-
-;; Helper for creating `vfmadd213pd` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213pd (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213pd x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213pd) x y z))
-
-
-;; Helper for creating `sqrtss` instructions.
-(decl x64_sqrtss (Xmm) Xmm)
-(rule (x64_sqrtss x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Sqrtss) x dst))))
-        dst))
-
-;; Helper for creating `sqrtsd` instructions.
-(decl x64_sqrtsd (Xmm) Xmm)
-(rule (x64_sqrtsd x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Sqrtsd) x dst))))
-        dst))
-
-;; Helper for creating `sqrtps` instructions.
-(decl x64_sqrtps (Xmm) Xmm)
-(rule (x64_sqrtps x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Sqrtps) x dst))))
-        dst))
-
-;; Helper for creating `sqrtpd` instructions.
-(decl x64_sqrtpd (Xmm) Xmm)
-(rule (x64_sqrtpd x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Sqrtpd) x dst))))
-        dst))
-
-;; Helper for creating `cvtss2sd` instructions.
-(decl x64_cvtss2sd (Xmm) Xmm)
-(rule (x64_cvtss2sd x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Cvtss2sd) x dst))))
-        dst))
-
-;; Helper for creating `cvtsd2ss` instructions.
-(decl x64_cvtsd2ss (Xmm) Xmm)
-(rule (x64_cvtsd2ss x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Cvtsd2ss) x dst))))
-        dst))
-
-;; Helper for creating `cvtdq2ps` instructions.
-(decl x64_cvtdq2ps (Xmm) Xmm)
-(rule (x64_cvtdq2ps x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Cvtdq2ps) x dst))))
-        dst))
-
-;; Helper for creating `cvtps2pd` instructions.
-(decl x64_cvtps2pd (Xmm) Xmm)
-(rule (x64_cvtps2pd x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Cvtps2pd) x dst))))
-        dst))
-
-;; Helper for creating `cvtpd2ps` instructions.
-(decl x64_cvtpd2ps (Xmm) Xmm)
-(rule (x64_cvtpd2ps x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Cvtpd2ps) x dst))))
-        dst))
-
-;; Helper for creating `cvtdq2pd` instructions.
-(decl x64_cvtdq2pd (Type Xmm) Xmm)
-(rule (x64_cvtdq2pd ty x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Cvtdq2pd) x dst))))
-        dst))
-
-;; Helper for creating `cvtsi2ss` instructions.
-(decl x64_cvtsi2ss (Type GprMem) Xmm)
-(rule (x64_cvtsi2ss ty x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (size OperandSize (raw_operand_size_of_type ty))
-            (_ Unit (emit (MInst.GprToXmm (SseOpcode.Cvtsi2ss) x dst size))))
-        dst))
-
-;; Helper for creating `cvtsi2sd` instructions.
-(decl x64_cvtsi2sd (Type GprMem) Xmm)
-(rule (x64_cvtsi2sd ty x)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (size OperandSize (raw_operand_size_of_type ty))
-            (_ Unit (emit (MInst.GprToXmm (SseOpcode.Cvtsi2sd) x dst size))))
-        dst))
-
-;; Helper for creating `cvttps2dq` instructions.
-(decl x64_cvttps2dq (Type XmmMem) Xmm)
-(rule (x64_cvttps2dq ty x)
-      (xmm_unary_rm_r (SseOpcode.Cvttps2dq) x))
-
-;; Helper for creating `cvttpd2dq` instructions.
-(decl x64_cvttpd2dq (XmmMem) Xmm)
-(rule (x64_cvttpd2dq x)
-      (xmm_unary_rm_r (SseOpcode.Cvttpd2dq) x))
-
-(decl cvt_u64_to_float_seq (Type Gpr) Xmm)
-(rule (cvt_u64_to_float_seq ty src)
-      (let ((size OperandSize (raw_operand_size_of_type ty))
-            (dst WritableXmm (temp_writable_xmm))
-            (tmp_gpr1 WritableGpr (temp_writable_gpr))
-            (tmp_gpr2 WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.CvtUint64ToFloatSeq size src dst tmp_gpr1 tmp_gpr2))))
-        dst))
-
-(decl cvt_float_to_uint_seq (Type Value bool) Gpr)
-(rule (cvt_float_to_uint_seq out_ty src @ (value_type src_ty) is_saturating)
-      (let ((out_size OperandSize (raw_operand_size_of_type out_ty))
-            (src_size OperandSize (raw_operand_size_of_type src_ty))
-
-            (dst WritableGpr (temp_writable_gpr))
-            (tmp_xmm WritableXmm (temp_writable_xmm))
-            (tmp_xmm2 WritableXmm (temp_writable_xmm))
-            (tmp_gpr WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.CvtFloatToUintSeq out_size src_size is_saturating src dst tmp_gpr tmp_xmm tmp_xmm2))))
-        dst))
-
-(decl cvt_float_to_sint_seq (Type Value bool) Gpr)
-(rule (cvt_float_to_sint_seq out_ty src @ (value_type src_ty) is_saturating)
-      (let ((out_size OperandSize (raw_operand_size_of_type out_ty))
-            (src_size OperandSize (raw_operand_size_of_type src_ty))
-
-            (dst WritableGpr (temp_writable_gpr))
-            (tmp_xmm WritableXmm (temp_writable_xmm))
-            (tmp_gpr WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.CvtFloatToSintSeq out_size src_size is_saturating src dst tmp_gpr tmp_xmm))))
-        dst))
-
-(decl fcvt_uint_mask_const () VCodeConstant)
-(extern constructor fcvt_uint_mask_const fcvt_uint_mask_const)
-
-(decl fcvt_uint_mask_high_const () VCodeConstant)
-(extern constructor fcvt_uint_mask_high_const fcvt_uint_mask_high_const)
-
-;; Helpers for creating `pcmpeq*` instructions.
-(decl x64_pcmpeq (Type Xmm XmmMem) Xmm)
-(rule (x64_pcmpeq $I8X16 x y) (x64_pcmpeqb x y))
-(rule (x64_pcmpeq $I16X8 x y) (x64_pcmpeqw x y))
-(rule (x64_pcmpeq $I32X4 x y) (x64_pcmpeqd x y))
-(rule (x64_pcmpeq $I64X2 x y) (x64_pcmpeqq x y))
-
-(decl x64_pcmpeqb (Xmm XmmMem) Xmm)
-(rule (x64_pcmpeqb x y) (xmm_rm_r $I8X16 (SseOpcode.Pcmpeqb) x y))
-(decl x64_pcmpeqw (Xmm XmmMem) Xmm)
-(rule (x64_pcmpeqw x y) (xmm_rm_r $I16X8 (SseOpcode.Pcmpeqw) x y))
-(decl x64_pcmpeqd (Xmm XmmMem) Xmm)
-(rule (x64_pcmpeqd x y) (xmm_rm_r $I32X4 (SseOpcode.Pcmpeqd) x y))
-(decl x64_pcmpeqq (Xmm XmmMem) Xmm)
-(rule (x64_pcmpeqq x y) (xmm_rm_r $I64X2 (SseOpcode.Pcmpeqq) x y))
-
-;; Helpers for creating `pcmpgt*` instructions.
-(decl x64_pcmpgt (Type Xmm XmmMem) Xmm)
-(rule (x64_pcmpgt $I8X16 x y) (x64_pcmpgtb x y))
-(rule (x64_pcmpgt $I16X8 x y) (x64_pcmpgtw x y))
-(rule (x64_pcmpgt $I32X4 x y) (x64_pcmpgtd x y))
-(rule (x64_pcmpgt $I64X2 x y) (x64_pcmpgtq x y))
-
-(decl x64_pcmpgtb (Xmm XmmMem) Xmm)
-(rule (x64_pcmpgtb x y) (xmm_rm_r $I8X16 (SseOpcode.Pcmpgtb) x y))
-(decl x64_pcmpgtw (Xmm XmmMem) Xmm)
-(rule (x64_pcmpgtw x y) (xmm_rm_r $I16X8 (SseOpcode.Pcmpgtw) x y))
-(decl x64_pcmpgtd (Xmm XmmMem) Xmm)
-(rule (x64_pcmpgtd x y) (xmm_rm_r $I32X4 (SseOpcode.Pcmpgtd) x y))
-(decl x64_pcmpgtq (Xmm XmmMem) Xmm)
-(rule (x64_pcmpgtq x y) (xmm_rm_r $I64X2 (SseOpcode.Pcmpgtq) x y))
-
-;; Helpers for read-modify-write ALU form (AluRM).
-(decl alu_rm (Type AluRmiROpcode Amode Gpr) SideEffectNoResult)
-(rule (alu_rm ty opcode src1_dst src2)
-      (let ((size OperandSize (operand_size_of_type_32_64 ty)))
-        (SideEffectNoResult.Inst (MInst.AluRM size opcode src1_dst src2))))
-
-(decl x64_add_mem (Type Amode Gpr) SideEffectNoResult)
-(rule (x64_add_mem ty addr val)
-      (alu_rm ty (AluRmiROpcode.Add) addr val))
-
-(decl x64_sub_mem (Type Amode Gpr) SideEffectNoResult)
-(rule (x64_sub_mem ty addr val)
-      (alu_rm ty (AluRmiROpcode.Sub) addr val))
-
-(decl x64_and_mem (Type Amode Gpr) SideEffectNoResult)
-(rule (x64_and_mem ty addr val)
-      (alu_rm ty (AluRmiROpcode.And) addr val))
-
-(decl x64_or_mem (Type Amode Gpr) SideEffectNoResult)
-(rule (x64_or_mem ty addr val)
-      (alu_rm ty (AluRmiROpcode.Or) addr val))
-
-(decl x64_xor_mem (Type Amode Gpr) SideEffectNoResult)
-(rule (x64_xor_mem ty addr val)
-      (alu_rm ty (AluRmiROpcode.Xor) addr val))
-
-;; Trap if the condition code supplied is set.
-(decl trap_if (CC TrapCode) ConsumesFlags)
-(rule (trap_if cc tc)
-      (ConsumesFlags.ConsumesFlagsSideEffect (MInst.TrapIf cc tc)))
-
-;; Trap if both of the condition codes supplied are set.
-(decl trap_if_and (CC CC TrapCode) ConsumesFlags)
-(rule (trap_if_and cc1 cc2 tc)
-      (ConsumesFlags.ConsumesFlagsSideEffect (MInst.TrapIfAnd cc1 cc2 tc)))
-
-;; Trap if either of the condition codes supplied are set.
-(decl trap_if_or (CC CC TrapCode) ConsumesFlags)
-(rule (trap_if_or cc1 cc2 tc)
-      (ConsumesFlags.ConsumesFlagsSideEffect (MInst.TrapIfOr cc1 cc2 tc)))
-
-(decl trap_if_icmp (IcmpCondResult TrapCode) SideEffectNoResult)
-(rule (trap_if_icmp (IcmpCondResult.Condition producer cc) tc)
-      (with_flags_side_effect producer (trap_if cc tc)))
-
-(decl trap_if_fcmp (FcmpCondResult TrapCode) SideEffectNoResult)
-(rule (trap_if_fcmp (FcmpCondResult.Condition producer cc) tc)
-      (with_flags_side_effect producer (trap_if cc tc)))
-(rule (trap_if_fcmp (FcmpCondResult.AndCondition producer cc1 cc2) tc)
-      (with_flags_side_effect producer (trap_if_and cc1 cc2 tc)))
-(rule (trap_if_fcmp (FcmpCondResult.OrCondition producer cc1 cc2) tc)
-      (with_flags_side_effect producer (trap_if_or cc1 cc2 tc)))
-
-;;;; Jumps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Unconditional jump.
-(decl jmp_known (MachLabel) SideEffectNoResult)
-(rule (jmp_known target)
-      (SideEffectNoResult.Inst (MInst.JmpKnown target)))
-
-(decl jmp_if (CC MachLabel) ConsumesFlags)
-(rule (jmp_if cc taken)
-      (ConsumesFlags.ConsumesFlagsSideEffect (MInst.JmpIf cc taken)))
-
-;; Conditional jump based on the condition code.
-(decl jmp_cond (CC MachLabel MachLabel) ConsumesFlags)
-(rule (jmp_cond cc taken not_taken)
-      (ConsumesFlags.ConsumesFlagsSideEffect (MInst.JmpCond cc taken not_taken)))
-
-;; Conditional jump based on the result of an icmp.
-(decl jmp_cond_icmp (IcmpCondResult MachLabel MachLabel) SideEffectNoResult)
-(rule (jmp_cond_icmp (IcmpCondResult.Condition producer cc) taken not_taken)
-      (with_flags_side_effect producer (jmp_cond cc taken not_taken)))
-
-;; Conditional jump based on the result of an fcmp.
-(decl jmp_cond_fcmp (FcmpCondResult MachLabel MachLabel) SideEffectNoResult)
-(rule (jmp_cond_fcmp (FcmpCondResult.Condition producer cc) taken not_taken)
-      (with_flags_side_effect producer (jmp_cond cc taken not_taken)))
-(rule (jmp_cond_fcmp (FcmpCondResult.AndCondition producer cc1 cc2) taken not_taken)
-      (with_flags_side_effect producer
-                              (consumes_flags_concat
-                                (jmp_if (cc_invert cc1) not_taken)
-                                (jmp_cond (cc_invert cc2) not_taken taken))))
-(rule (jmp_cond_fcmp (FcmpCondResult.OrCondition producer cc1 cc2) taken not_taken)
-      (with_flags_side_effect producer
-                              (consumes_flags_concat
-                                (jmp_if cc1 taken)
-                                (jmp_cond cc2 taken not_taken))))
-
-;; Emit the compound instruction that does:
-;;
-;; lea $jt, %rA
-;; movsbl [%rA, %rIndex, 2], %rB
-;; add %rB, %rA
-;; j *%rA
-;; [jt entries]
-;;
-;; This must be *one* instruction in the vcode because we cannot allow regalloc
-;; to insert any spills/fills in the middle of the sequence; otherwise, the
-;; lea PC-rel offset to the jumptable would be incorrect.  (The alternative
-;; is to introduce a relocation pass for inlined jumptables, which is much
-;; worse.)
-(decl jmp_table_seq (Type Gpr MachLabel BoxVecMachLabel) SideEffectNoResult)
-(rule (jmp_table_seq ty idx default_target jt_targets)
-      (let (;; This temporary is used as a signed integer of 64-bits (to hold
-            ;; addresses).
-            (tmp1 WritableGpr (temp_writable_gpr))
-
-            ;; This temporary is used as a signed integer of 32-bits (for the
-            ;; wasm-table index) and then 64-bits (address addend). The small
-            ;; lie about the I64 type is benign, since the temporary is dead
-            ;; after this instruction (and its Cranelift type is thus unused).
-            (tmp2 WritableGpr (temp_writable_gpr))
-
-            (size OperandSize (raw_operand_size_of_type ty))
-
-            (jt_size u32 (jump_table_size jt_targets)))
-
-        (with_flags_side_effect
-          (x64_cmp size (RegMemImm.Imm jt_size) idx)
-          (ConsumesFlags.ConsumesFlagsSideEffect
-            (MInst.JmpTableSeq idx tmp1 tmp2 default_target jt_targets)))))
-
-;;;; iadd_pairwise constants ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl iadd_pairwise_mul_const_16 () VCodeConstant)
-(extern constructor iadd_pairwise_mul_const_16 iadd_pairwise_mul_const_16)
-
-(decl iadd_pairwise_mul_const_32 () VCodeConstant)
-(extern constructor iadd_pairwise_mul_const_32 iadd_pairwise_mul_const_32)
-
-(decl iadd_pairwise_xor_const_32 () VCodeConstant)
-(extern constructor iadd_pairwise_xor_const_32 iadd_pairwise_xor_const_32)
-
-(decl iadd_pairwise_addd_const_32 () VCodeConstant)
-(extern constructor iadd_pairwise_addd_const_32 iadd_pairwise_addd_const_32)
-
-;;;; snarrow constants ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl snarrow_umax_mask () VCodeConstant)
-(extern constructor snarrow_umax_mask snarrow_umax_mask)
-
-;;;; Comparisons ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(type IcmpCondResult (enum (Condition (producer ProducesFlags) (cc CC))))
-
-(decl icmp_cond_result (ProducesFlags CC) IcmpCondResult)
-(rule (icmp_cond_result producer cc) (IcmpCondResult.Condition producer cc))
-
-(decl invert_icmp_cond_result (IcmpCondResult) IcmpCondResult)
-(rule (invert_icmp_cond_result (IcmpCondResult.Condition producer cc))
-      (icmp_cond_result producer (cc_invert cc)))
-
-;; Lower an Icmp result into a boolean value in a register.
-(decl lower_icmp_bool (IcmpCondResult) ValueRegs)
-(rule (lower_icmp_bool (IcmpCondResult.Condition producer cc))
-      (with_flags producer (x64_setcc cc)))
-
-;; Emit a conditional move based on the result of an icmp.
-(decl select_icmp (IcmpCondResult Value Value) ValueRegs)
-
-;; Ensure that we put the `x` argument into a register for single-register
-;; gpr-typed arguments, as we rely on this for the legalization of heap_addr and
-;; loading easily computed constants (like 0) from memory is too expensive.
-(rule 1 (select_icmp (IcmpCondResult.Condition producer cc) x @ (value_type (is_single_register_gpr_type ty)) y)
-      (with_flags producer (cmove ty cc (put_in_gpr x) y)))
-
-;; Otherwise, fall back on the behavior of `cmove_from_values`.
-(rule 0 (select_icmp (IcmpCondResult.Condition producer cc) x @ (value_type ty) y)
-      (with_flags producer (cmove_from_values ty cc x y)))
-
-(decl emit_cmp (IntCC Value Value) IcmpCondResult)
-
-;; For GPR-held values we only need to emit `CMP + SETCC`. We rely here on
-;; Cranelift's verification that `a` and `b` are of the same type.
-;; Unfortunately for clarity, the registers are flipped here (TODO).
-(rule 0 (emit_cmp cc a @ (value_type ty) b)
-      (let ((size OperandSize (raw_operand_size_of_type ty)))
-        (icmp_cond_result (x64_cmp size b a) cc)))
-
-;; As a special case, reverse the arguments to the comparison when the LHS is a
-;; constant. This ensures that we avoid moving the constant into a register when
-;; performing the comparison.
-(rule 1 (emit_cmp cc (and (simm32_from_value a) (value_type ty)) b)
-      (let ((size OperandSize (raw_operand_size_of_type ty)))
-        (icmp_cond_result (x64_cmp size a b) (intcc_reverse cc))))
-
-;; For I128 values (held in two GPRs), the instruction sequences depend on what
-;; kind of condition is tested.
-(rule 3 (emit_cmp (IntCC.Equal) a @ (value_type $I128) b)
-      (let ((a_lo Gpr (value_regs_get_gpr a 0))
-            (a_hi Gpr (value_regs_get_gpr a 1))
-            (b_lo Gpr (value_regs_get_gpr b 0))
-            (b_hi Gpr (value_regs_get_gpr b 1))
-            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_lo a_lo) (x64_setcc (CC.Z))))
-            (cmp_hi Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_hi a_hi) (x64_setcc (CC.Z))))
-            ;; At this point, `cmp_lo` and `cmp_hi` contain either 0 or 1 in the
-            ;; lowest 8 bits--`SETcc` guarantees this. The upper bits may be
-            ;; unchanged so we must compare against 1 below; this instruction
-            ;; combines `cmp_lo` and `cmp_hi` for that final comparison.
-            (cmp Reg (x64_and $I64 cmp_lo cmp_hi)))
-        ;; We must compare one more time against the immediate value 1 to
-        ;; check if both `cmp_lo` and `cmp_hi` are true. If `cmp AND 1 == 0`
-        ;; then the `ZF` will be set (see `TEST` definition); if either of
-        ;; the halves `AND`s to 0, they were not equal, therefore we `SETcc`
-        ;; with `NZ`.
-        (icmp_cond_result
-          (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) cmp)
-          (CC.NZ))))
-
-(rule 3 (emit_cmp (IntCC.NotEqual) a @ (value_type $I128) b)
-      (let ((a_lo Gpr (value_regs_get_gpr a 0))
-            (a_hi Gpr (value_regs_get_gpr a 1))
-            (b_lo Gpr (value_regs_get_gpr b 0))
-            (b_hi Gpr (value_regs_get_gpr b 1))
-            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_lo a_lo) (x64_setcc (CC.NZ))))
-            (cmp_hi Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_hi a_hi) (x64_setcc (CC.NZ))))
-            ;; See comments for `IntCC.Equal`.
-            (cmp Reg (x64_or $I64 cmp_lo cmp_hi)))
-           (icmp_cond_result
-             (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) cmp)
-             (CC.NZ))))
-
-;; Result = (a_hi <> b_hi) ||
-;;          (a_hi == b_hi && a_lo <> b_lo)
-(rule 2 (emit_cmp cc a @ (value_type $I128) b)
-      (let ((a_lo Gpr (value_regs_get_gpr a 0))
-            (a_hi Gpr (value_regs_get_gpr a 1))
-            (b_lo Gpr (value_regs_get_gpr b 0))
-            (b_hi Gpr (value_regs_get_gpr b 1))
-            (cmp_hi ValueRegs (with_flags (x64_cmp (OperandSize.Size64) b_hi a_hi)
-                                       (consumes_flags_concat
-                                                 (x64_setcc (intcc_without_eq cc))
-                                                 (x64_setcc (CC.Z)))))
-            (cc_hi Reg (value_regs_get cmp_hi 0))
-            (eq_hi Reg (value_regs_get cmp_hi 1))
-
-            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_lo a_lo)
-                                        (x64_setcc (intcc_unsigned cc))))
-
-            (res_lo Reg (x64_and $I64 eq_hi cmp_lo))
-            (res Reg (x64_or $I64 cc_hi res_lo)))
-        (icmp_cond_result
-          (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) res)
-          (CC.NZ))))
-
-(type FcmpCondResult
-      (enum
-        ;; The given condition code must be set.
-        (Condition (producer ProducesFlags) (cc CC))
-
-        ;; Both condition codes must be set.
-        (AndCondition (producer ProducesFlags) (cc1 CC) (cc2 CC))
-
-        ;; Either of the conditions codes must be set.
-        (OrCondition (producer ProducesFlags) (cc1 CC) (cc2 CC))))
-
-;; Lower a FcmpCondResult to a boolean value in a register.
-(decl lower_fcmp_bool (FcmpCondResult) ValueRegs)
-
-(rule (lower_fcmp_bool (FcmpCondResult.Condition producer cc))
-      (with_flags producer (x64_setcc cc)))
-
-(rule (lower_fcmp_bool (FcmpCondResult.AndCondition producer cc1 cc2))
-      (let ((maybe ValueRegs (with_flags producer
-                                         (consumes_flags_concat
-                                           (x64_setcc cc1)
-                                           (x64_setcc cc2))))
-            (maybe0 Gpr (value_regs_get_gpr maybe 0))
-            (maybe1 Gpr (value_regs_get_gpr maybe 1)))
-        (value_reg (x64_and $I8 maybe0 maybe1))))
-
-(rule (lower_fcmp_bool (FcmpCondResult.OrCondition producer cc1 cc2))
-      (let ((maybe ValueRegs (with_flags producer
-                                         (consumes_flags_concat
-                                           (x64_setcc cc1)
-                                           (x64_setcc cc2))))
-            (maybe0 Gpr (value_regs_get_gpr maybe 0))
-            (maybe1 Gpr (value_regs_get_gpr maybe 1)))
-        (value_reg (x64_or $I8 maybe0 maybe1))))
-
-;; CLIF's `fcmp` instruction always operates on XMM registers--both scalar and
-;; vector. For the scalar versions, we use the flag-setting behavior of the
-;; `UCOMIS*` instruction to `SETcc` a 0 or 1 in a GPR register. Note that CLIF's
-;; `select` uses the same kind of flag-setting behavior but chooses values other
-;; than 0 or 1.
-;;
-;; Checking the result of `UCOMIS*` is unfortunately difficult in some cases
-;; because we do not have `SETcc` instructions that explicitly check
-;; simultaneously for the condition (i.e., `eq`, `le`, `gt`, etc.) *and*
-;; orderedness. Instead, we must check the flags multiple times. The UCOMIS*
-;; documentation (see Intel's Software Developer's Manual, volume 2, chapter 4)
-;; is helpful:
-;;  - unordered assigns    Z = 1, P = 1, C = 1
-;;  - greater than assigns Z = 0, P = 0, C = 0
-;;  - less than assigns    Z = 0, P = 0, C = 1
-;;  - equal assigns        Z = 1, P = 0, C = 0
-(decl emit_fcmp (FloatCC Value Value) FcmpCondResult)
-
-(rule (emit_fcmp (FloatCC.Equal) a @ (value_type (ty_scalar_float _)) b)
-      (FcmpCondResult.AndCondition (x64_ucomis b a) (CC.NP) (CC.Z)))
-
-(rule (emit_fcmp (FloatCC.NotEqual) a @ (value_type (ty_scalar_float _)) b)
-      (FcmpCondResult.OrCondition (x64_ucomis b a) (CC.P) (CC.NZ)))
-
-;; Some scalar lowerings correspond to one condition code.
-
-(rule (emit_fcmp (FloatCC.Ordered) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.NP)))
-(rule (emit_fcmp (FloatCC.Unordered) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.P)))
-(rule (emit_fcmp (FloatCC.OrderedNotEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.NZ)))
-(rule (emit_fcmp (FloatCC.UnorderedOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.Z)))
-(rule (emit_fcmp (FloatCC.GreaterThan) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.NBE)))
-(rule (emit_fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.NB)))
-(rule (emit_fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.B)))
-(rule (emit_fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.BE)))
-
-;; Other scalar lowerings are made possible by flipping the operands and
-;; reversing the condition code.
-
-(rule (emit_fcmp (FloatCC.LessThan) a @ (value_type (ty_scalar_float ty)) b)
-      ;; Same flags as `GreaterThan`.
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.NBE)))
-(rule (emit_fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      ;; Same flags as `GreaterThanOrEqual`.
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.NB)))
-(rule (emit_fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_scalar_float ty)) b)
-      ;; Same flags as `UnorderedOrLessThan`.
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.B)))
-(rule (emit_fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      ;; Same flags as `UnorderedOrLessThanOrEqual`.
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.BE)))
-
-;;;; Type Guards ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; A type guard for matching ints and bools up to 64 bits, or 64 bit references.
-(decl ty_int_bool_or_ref () Type)
-(extern extractor ty_int_bool_or_ref ty_int_bool_or_ref)
-
-;;;; Atomics ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl x64_mfence () SideEffectNoResult)
-(rule (x64_mfence)
-      (SideEffectNoResult.Inst (MInst.Fence (FenceKind.MFence))))
-
-(decl x64_cmpxchg (Type Gpr Gpr SyntheticAmode) Gpr)
-(rule (x64_cmpxchg ty expected replacement addr)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.LockCmpxchg ty replacement expected addr dst))))
-        dst))
-
-(decl x64_atomic_rmw_seq (Type MachAtomicRmwOp SyntheticAmode Gpr) Gpr)
-(rule (x64_atomic_rmw_seq ty op mem input)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (tmp WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.AtomicRmwSeq ty op mem input tmp dst))))
-        dst))
-
-;; CLIF IR has one enumeration for atomic operations (`AtomicRmwOp`) while the
-;; mach backend has another (`MachAtomicRmwOp`)--this converts one to the other.
-(type MachAtomicRmwOp extern (enum))
-(decl atomic_rmw_op_to_mach_atomic_rmw_op (AtomicRmwOp) MachAtomicRmwOp)
-(extern constructor atomic_rmw_op_to_mach_atomic_rmw_op atomic_rmw_op_to_mach_atomic_rmw_op)
-
-;;;; Casting ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl bitcast_xmm_to_gpr (Type Xmm) Gpr)
-(rule (bitcast_xmm_to_gpr $F32 src)
-      (xmm_to_gpr (SseOpcode.Movd) src (OperandSize.Size32)))
-(rule (bitcast_xmm_to_gpr $F64 src)
-      (xmm_to_gpr (SseOpcode.Movq) src (OperandSize.Size64)))
-
-(decl bitcast_gpr_to_xmm (Type Gpr) Xmm)
-(rule (bitcast_gpr_to_xmm $I32 src)
-      (gpr_to_xmm (SseOpcode.Movd) src (OperandSize.Size32)))
-(rule (bitcast_gpr_to_xmm $I64 src)
-      (gpr_to_xmm (SseOpcode.Movq) src (OperandSize.Size64)))
-
-;;;; Stack Addresses ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl stack_addr_impl (StackSlot Offset32) Gpr)
-(rule (stack_addr_impl stack_slot offset)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (abi_stackslot_addr dst stack_slot offset))))
-        dst))
-
-;;;; Division/Remainders ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl emit_div_or_rem (DivOrRemKind Type WritableGpr Gpr Gpr) Unit)
-(extern constructor emit_div_or_rem emit_div_or_rem)
-
-(decl div_or_rem (DivOrRemKind Value Value) Gpr)
-(rule (div_or_rem kind a @ (value_type ty) b)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit_div_or_rem kind ty dst a b)))
-        dst))
-
-;;;; Pinned Register ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl read_pinned_gpr () Gpr)
-(rule (read_pinned_gpr)
-      (mov_from_preg (preg_pinned)))
-
-(decl write_pinned_gpr (Gpr) SideEffectNoResult)
-(rule (write_pinned_gpr val)
-      (mov_to_preg (preg_pinned) val))
-
-;;;; Shuffle ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Produce a mask suitable for use with `pshufb` for permuting the argument to
-;; shuffle, when the arguments are the same (i.e. `shuffle a a mask`). This will
-;; map all indices in the range 0..31 to the range 0..15.
-(decl shuffle_0_31_mask (VecMask) VCodeConstant)
-(extern constructor shuffle_0_31_mask shuffle_0_31_mask)
-
-;; Produce a mask suitable for use with `pshufb` for permuting the lhs of a
-;; `shuffle` operation (lanes 0-15).
-(decl shuffle_0_15_mask (VecMask) VCodeConstant)
-(extern constructor shuffle_0_15_mask shuffle_0_15_mask)
-
-;; Produce a mask suitable for use with `pshufb` for permuting the rhs of a
-;; `shuffle` operation (lanes 16-31).
-(decl shuffle_16_31_mask (VecMask) VCodeConstant)
-(extern constructor shuffle_16_31_mask shuffle_16_31_mask)
-
-;; Produce a permutation suitable for use with `vpermi2b`, for permuting two
-;; I8X16 vectors simultaneously.
-;;
-;; NOTE: `vpermi2b` will mask the indices in each lane to 5 bits when indexing
-;; into vectors, so this constructor makes no effort to handle indices that are
-;; larger than 31. If you are lowering a clif opcode like `shuffle` that has
-;; special behavior for out of bounds indices (emitting a `0` in the resulting
-;; vector in the case of `shuffle`) you'll need to handle that behavior
-;; separately.
-(decl perm_from_mask (VecMask) VCodeConstant)
-(extern constructor perm_from_mask perm_from_mask)
-
-;; If the mask that would be given to `shuffle` contains any out-of-bounds
-;; indices, return a mask that will zero those.
-(decl perm_from_mask_with_zeros (VCodeConstant VCodeConstant) VecMask)
-(extern extractor perm_from_mask_with_zeros perm_from_mask_with_zeros)
-
-;;;; Swizzle ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Create a mask for zeroing out-of-bounds lanes of the swizzle mask.
-(decl swizzle_zero_mask () VCodeConstant)
-(extern constructor swizzle_zero_mask swizzle_zero_mask)
-
-;;;; TLS Values ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Helper for emitting ElfTlsGetAddr.
-(decl elf_tls_get_addr (ExternalName) Gpr)
-(rule (elf_tls_get_addr name)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.ElfTlsGetAddr name dst))))
-        dst))
-
-;; Helper for emitting MachOTlsGetAddr.
-(decl macho_tls_get_addr (ExternalName) Gpr)
-(rule (macho_tls_get_addr name)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.MachOTlsGetAddr name dst))))
-        dst))
-
-;; Helper for emitting CoffTlsGetAddr.
-(decl coff_tls_get_addr (ExternalName) Gpr)
-(rule (coff_tls_get_addr name)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (tmp WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.CoffTlsGetAddr name dst tmp))))
-        dst))
-
-;;;; sqmul_round_sat ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl sqmul_round_sat_mask () VCodeConstant)
-(extern constructor sqmul_round_sat_mask sqmul_round_sat_mask)
-
-;;;; uunarrow ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(decl uunarrow_umax_mask () VCodeConstant)
-(extern constructor uunarrow_umax_mask uunarrow_umax_mask)
-
-(decl uunarrow_uint_mask () VCodeConstant)
-(extern constructor uunarrow_uint_mask uunarrow_uint_mask)
-
-;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(convert Gpr InstOutput output_gpr)
-(convert Value Gpr put_in_gpr)
-(convert Value GprMem put_in_gpr_mem)
-(convert Value GprMemImm put_in_gpr_mem_imm)
-(convert Value RegMem put_in_reg_mem)
-(convert Value RegMemImm put_in_reg_mem_imm)
-(convert Gpr GprMemImm gpr_to_gpr_mem_imm)
-(convert Gpr GprMem gpr_to_gpr_mem)
-(convert Gpr Reg gpr_to_reg)
-(convert GprMem RegMem gpr_mem_to_reg_mem)
-(convert Reg Gpr gpr_new)
-(convert WritableGpr Gpr writable_gpr_to_gpr)
-(convert RegMemImm GprMemImm gpr_mem_imm_new)
-(convert RegMem GprMem reg_mem_to_gpr_mem)
-(convert RegMem RegMemImm reg_mem_to_reg_mem_imm)
-(convert Reg GprMem reg_to_gpr_mem)
-(convert Reg GprMemImm reg_to_gpr_mem_imm)
-(convert WritableGpr WritableReg writable_gpr_to_reg)
-(convert WritableGpr Reg writable_gpr_to_r_reg)
-(convert WritableGpr GprMem writable_gpr_to_gpr_mem)
-(convert WritableGpr ValueRegs writable_gpr_to_value_regs)
-
-(convert Xmm InstOutput output_xmm)
-(convert Value Xmm put_in_xmm)
-(convert Value XmmMem put_in_xmm_mem)
-(convert Value XmmMemImm put_in_xmm_mem_imm)
-(convert Xmm Reg xmm_to_reg)
-(convert Xmm RegMem xmm_to_reg_mem)
-(convert Reg Xmm xmm_new)
-(convert Reg XmmMem reg_to_xmm_mem)
-(convert Reg RegMemImm reg_to_reg_mem_imm)
-(convert RegMem XmmMem reg_mem_to_xmm_mem)
-(convert RegMemImm XmmMemImm mov_rmi_to_xmm)
-(convert Xmm XmmMem xmm_to_xmm_mem)
-(convert Xmm XmmMemImm xmm_to_xmm_mem_imm)
-(convert XmmMem RegMem xmm_mem_to_reg_mem)
-(convert WritableXmm Xmm writable_xmm_to_xmm)
-(convert WritableXmm WritableReg writable_xmm_to_reg)
-(convert WritableXmm Reg writable_xmm_to_r_reg)
-(convert WritableXmm XmmMem writable_xmm_to_xmm_mem)
-(convert WritableXmm ValueRegs writable_xmm_to_value_regs)
-
-(convert Gpr Imm8Gpr gpr_to_imm8_gpr)
-(convert Imm8Reg Imm8Gpr imm8_reg_to_imm8_gpr)
-
-(convert Amode SyntheticAmode amode_to_synthetic_amode)
-(convert Amode GprMem amode_to_gpr_mem)
-(convert SyntheticAmode GprMem synthetic_amode_to_gpr_mem)
-(convert Amode XmmMem amode_to_xmm_mem)
-(convert SyntheticAmode XmmMem synthetic_amode_to_xmm_mem)
-
-(convert IntCC CC intcc_to_cc)
-(convert AtomicRmwOp MachAtomicRmwOp atomic_rmw_op_to_mach_atomic_rmw_op)
-
-(decl reg_to_xmm_mem (Reg) XmmMem)
-(rule (reg_to_xmm_mem r)
-      (xmm_to_xmm_mem (xmm_new r)))
-(decl xmm_to_reg_mem (Reg) XmmMem)
-(rule (xmm_to_reg_mem r)
-      (RegMem.Reg (xmm_to_reg r)))
-
-(decl writable_gpr_to_r_reg (WritableGpr) Reg)
-(rule (writable_gpr_to_r_reg w_gpr)
-      (writable_reg_to_reg (writable_gpr_to_reg w_gpr)))
-(decl writable_gpr_to_gpr_mem (WritableGpr) GprMem)
-(rule (writable_gpr_to_gpr_mem w_gpr)
-      (gpr_to_gpr_mem w_gpr))
-(decl writable_gpr_to_value_regs (WritableGpr) ValueRegs)
-(rule (writable_gpr_to_value_regs w_gpr)
-      (value_reg w_gpr))
-(decl writable_xmm_to_r_reg (WritableXmm) Reg)
-(rule (writable_xmm_to_r_reg w_xmm)
-      (writable_reg_to_reg (writable_xmm_to_reg w_xmm)))
-(decl writable_xmm_to_xmm_mem (WritableXmm) XmmMem)
-(rule (writable_xmm_to_xmm_mem w_xmm)
-      (xmm_to_xmm_mem (writable_xmm_to_xmm w_xmm)))
-(decl writable_xmm_to_value_regs (WritableXmm) ValueRegs)
-(rule (writable_xmm_to_value_regs w_xmm)
-      (value_reg w_xmm))
-
-(decl synthetic_amode_to_gpr_mem (SyntheticAmode) GprMem)
-(decl amode_to_gpr_mem (Amode) GprMem)
-(rule (amode_to_gpr_mem amode)
-      (amode_to_synthetic_amode amode))
-(rule (synthetic_amode_to_gpr_mem amode)
-      (synthetic_amode_to_reg_mem amode))
-(decl amode_to_xmm_mem (Amode) XmmMem)
-(rule (amode_to_xmm_mem amode)
-      (amode_to_synthetic_amode amode))
-(decl synthetic_amode_to_xmm_mem (SyntheticAmode) XmmMem)
-(rule (synthetic_amode_to_xmm_mem amode)
-      (synthetic_amode_to_reg_mem amode))
-(decl const_to_synthetic_amode (VCodeConstant) SyntheticAmode)
-(extern constructor const_to_synthetic_amode const_to_synthetic_amode)
-
-;; Helper for creating `MovFromPReg` instructions.
-(decl mov_from_preg (PReg) Reg)
-(rule (mov_from_preg preg)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.MovFromPReg preg dst))))
-        dst))
-
-(decl mov_to_preg (PReg Gpr) SideEffectNoResult)
-(rule (mov_to_preg dst src)
-      (SideEffectNoResult.Inst (MInst.MovToPReg src dst)))
-
-(decl preg_rbp () PReg)
-(extern constructor preg_rbp preg_rbp)
-
-(decl preg_rsp () PReg)
-(extern constructor preg_rsp preg_rsp)
-
-(decl preg_pinned () PReg)
-(extern constructor preg_pinned preg_pinned)
-
-(decl x64_rbp () Reg)
-(rule (x64_rbp)
-      (mov_from_preg (preg_rbp)))
-
-(decl x64_rsp () Reg)
-(rule (x64_rsp)
-      (mov_from_preg (preg_rsp)))
-
-;;;; Helpers for Emitting LibCalls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(type LibCall extern
-      (enum
-        FmaF32
-        FmaF64
-        CeilF32
-        CeilF64
-        FloorF32
-        FloorF64
-        NearestF32
-        NearestF64
-        TruncF32
-        TruncF64))
-
-(decl libcall_1 (LibCall Reg) Reg)
-(extern constructor libcall_1 libcall_1)
-
-(decl libcall_3 (LibCall Reg Reg Reg) Reg)
-(extern constructor libcall_3 libcall_3)