rubycc 1.0.0 → 1.1.0

data/lib/rubycc/backend/x86_64.rb

@@ -1,13 +1,22 @@
 # frozen_string_literal: true
 
 require_relative "../ir/ir"
+require_relative "../ir/analysis"
+require_relative "../ir/promotion"
+require_relative "../ir/simplify"
+require_relative "slot_residency"
 
 module Rubycc
   module Backend
     # x86_64 code generator using a spill-everything strategy: every virtual
     # register lives in its own 8-byte stack slot at [rbp - 8*(n+1)], and each
     # IR instruction loads its operands into eax/ecx, computes, and stores the
-    # result back. Arithmetic on a 4-byte-or-narrower type stays 32-bit (using
+    # result back. Two kinds of value are exempt: a transient, which stays in
+    # the register its producer left it in (IR::Simplify#transient_vregs), and a
+    # promoted one, which owns a callee-saved register for the whole function
+    # and so has no slot at all (IR::Promotion, #promotion_assignment).
+    #
+    # Arithmetic on a 4-byte-or-narrower type stays 32-bit (using
     # eax/ecx), whose natural wrap-around reproduces C's semantics for free;
     # `long`/`unsigned long`/pointer arithmetic is 64-bit (a REX.W prefix,
     # selected by an IR op's size == 8). Slots are always read and written 64
@@ -47,6 +56,13 @@ module Rubycc
     # argument registers into a 176-byte register-save area so __builtin_va_arg
     # can reach the variable part.
     class X86_64
+      # Every slot access goes through #load_reg / #store_reg, which use this to
+      # skip a reload of a value the previous instruction has just left in a
+      # register. See slot_residency.rb for why "nothing has been emitted since"
+      # is a sufficient safety condition, and #emit_instruction's :label case
+      # for the one state change it cannot see.
+      include SlotResidency
+
       # Result of compiling one function: `bytes` is the machine code (an
       # ASCII-8BIT String), `symbols` is an array of
       # { name:, offset:, size: } describing the emitted function symbols, and
@@ -90,6 +106,32 @@ module Rubycc
       # post-alloca rsp as the block's base address.
       RSP = 4
 
+      # The registers whole-function promotion hands out, in the order it hands
+      # them out (see IR::Promotion and #promotion_assignment). All five are
+      # callee-saved under System V AMD64 (psABI 3.2.1: rbx, rbp and r12..r15
+      # are preserved across a call), which is what lets a promoted value stay
+      # put across one — and none of them is used for anything else here, the
+      # scratch, argument and return registers all being caller-saved. rbp is
+      # excluded, being the frame pointer every slot is addressed from.
+      #
+      # rbx is register 3 and needs no REX extension; r12..r15 have their high
+      # bit carried by REX.R when they land in a ModR/M reg field, by REX.B when
+      # they land in an rm field (or a SIB base, or an opcode's low three bits)
+      # and by REX.X in a SIB index — every one of which they do land in, since
+      # a promoted value is named where it lives rather than moved to a scratch
+      # register first. #emit_rex is where all four bits are decided.
+      #
+      # None of these is a scratch, argument or return register, so no residency
+      # the SlotResidency table keeps is ever keyed by one; that is what lets an
+      # instruction writing a promoted register leave the table standing (see
+      # SlotResidency#note_register_clobbered).
+      RBX = 3
+      R12 = 12
+      R13 = 13
+      R14 = 14
+      R15 = 15
+      PROMOTION_REGISTERS = [RBX, R12, R13, R14, R15].freeze
+
       # The two vector (xmm) scratch registers the floating ops use. Their
       # numbers 0/1 double as the ModR/M reg/rm fields, so no REX.R is ever
       # needed to name them. Every floating value round-trips through a GP stack
@@ -108,6 +150,11 @@ module Rubycc
       GP_RETURN_REGISTERS = [EAX, EDX].freeze
       SSE_RETURN_REGISTERS = [XMM0, XMM1].freeze
 
+      # :scaled_add's element size -> the SIB byte's two-bit scale field. The
+      # four sizes are the only ones the encoding can name, which is exactly the
+      # set IR::Simplify fuses a subscript for.
+      SIB_SCALES = { 1 => 0, 2 => 1, 4 => 2, 8 => 3 }.freeze
+
       # IR comparison op -> setcc opcode (second byte of the 0F 9x encoding).
       # The result is materialized into eax as an int 0/1 by movzx. The signed
       # forms (setl/setle/setg/setge) test the sign/overflow flags; the unsigned
@@ -126,8 +173,28 @@ module Rubycc
         uge: 0x93  # setae  (above or equal, unsigned >=)
       }.freeze
 
-      def compile(ir_func)
+      # `analysis` is the census of `ir_func`'s instruction list (IR::Analysis).
+      # The compiler passes the one IR::Simplify already took on its way
+      # through; a caller that hands over a function directly gets it taken
+      # here.
+      def compile(ir_func, analysis = IR::Analysis.of(ir_func))
         @code = +"".b
+        reset_slot_residency
+        # Slots this function never has to write: an expression temporary whose
+        # one reader is the instruction right behind its producer stays in the
+        # register it was computed in (see IR::Simplify#transient_flags and
+        # #store_reg). The answer is a property of the instruction list, so it
+        # is taken once here rather than rediscovered per instruction, and it
+        # arrives as an array indexed by vreg number — the numbers are dense and
+        # small, and this is asked on every store the backend emits.
+        @transient = analysis.transient
+        # Slots this function does not have at all: a promoted value lives in
+        # one callee-saved register from the prologue to every ret, so its slot
+        # is never read, written or named (see #promotion_assignment, #load_reg
+        # and #store_reg). Like @transient this is a property of the instruction
+        # list, decided once here, and indexed by vreg number for the same
+        # reason.
+        @promoted = promotion_assignment(ir_func, analysis)
         # Control-flow bookkeeping: `@labels` maps a label id to its resolved
         # code offset; `@fixups` collects [patch_offset, label_id] pairs whose
         # rel32 field is overwritten once every label offset is known.
@@ -154,12 +221,31 @@ module Rubycc
 
       private
 
+      # Binds the best promotion candidates to PROMOTION_REGISTERS in order,
+      # returning the vreg -> register table the whole backend consults (an
+      # array; a vreg that owns no register indexes nil). Taking the first few
+      # of an ordered list is the entire allocation: one register each, held for
+      # the function's length, so there is no interference to resolve and
+      # nothing to undo when the registers run out — the candidates past the
+      # fifth simply keep their slots.
+      #
+      # @promoted_registers is the same binding in assignment order, which is
+      # the order the save slots are laid out in (#emit_prologue).
+      def promotion_assignment(ir_func, analysis)
+        vregs = IR::Promotion.candidates(ir_func, analysis).first(PROMOTION_REGISTERS.size)
+        @promoted_registers = PROMOTION_REGISTERS.first(vregs.size)
+        promoted = Array.new(ir_func.vreg_count)
+        vregs.each_with_index { |vreg, index| promoted[vreg] = PROMOTION_REGISTERS[index] }
+        promoted
+      end
+
       # Frame layout, from rbp downward: first the virtual-register slots
       # (8 bytes each, rounded up to a 16-byte region), then the stack objects,
-      # and last — for a variadic function — a 176-byte register-save area below
-      # everything else. Each object is placed at a 16-byte-aligned size below
-      # the previous one, and @object_offsets[id] records the rbp-relative
-      # displacement of the object's base (its lowest address, i.e. element 0).
+      # then one 8-byte save slot per promoted register, and last — for a
+      # variadic function — a 176-byte register-save area below everything else.
+      # Each object is placed at a 16-byte-aligned size below the previous one,
+      # and @object_offsets[id] records the rbp-relative displacement of the
+      # object's base (its lowest address, i.e. element 0).
       def emit_prologue(vreg_count, param_count, param_kinds, stack_objects, variadic)
         vreg_region = align16(vreg_count * 8)
         @object_offsets = []
@@ -168,6 +254,17 @@ module Rubycc
           running += align16(object_size)
           @object_offsets << -running
         end
+        # One save slot per promoted register. The registers are callee-saved,
+        # so a function that uses one has to give it back exactly as it found
+        # it; saving them into the frame with mov rather than with push keeps
+        # rsp's 16-byte alignment — which every call depends on — a property of
+        # the single `sub` below instead of the parity of the register count.
+        # They are laid out in assignment order, and their number is a function
+        # of the instruction list, so the frame stays deterministic (N4).
+        @promoted_saves = @promoted_registers.map do |reg|
+          running += 8
+          [reg, -running]
+        end
         # A variadic function reserves a 176-byte register-save area at the very
         # bottom of the frame; :va_start points reg_save_area here and the
         # prologue spills the six integer argument registers (48 bytes) followed
@@ -184,6 +281,10 @@ module Rubycc
         emit(0x48, 0x89, 0xE5)              # mov rbp, rsp
         emit(0x48, 0x81, 0xEC)              # sub rsp, imm32
         emit_bytes([frame_size].pack("L<"))
+        # Before anything writes a promoted register — which the parameter
+        # spilling right below is the first thing to do, a promoted parameter
+        # being moved straight from its incoming argument register.
+        emit_save_promoted_registers
         spill_parameters(param_kinds)
         if variadic
           emit_save_gp_registers
@@ -191,6 +292,31 @@ module Rubycc
         end
       end
 
+      # Saves the promoted registers into their frame slots, "mov [rbp+disp],
+      # r64" apiece. REX.W widens the move and REX.R extends r12..r15, whose low
+      # three bits go into the ModR/M reg field; the rm field is rbp's, so no SIB
+      # byte is involved on either side.
+      def emit_save_promoted_registers
+        @promoted_saves.each do |reg, disp|
+          emit(0x48 | (reg >= 8 ? 0x04 : 0)) # REX.W (+ REX.R for r12..r15)
+          emit(0x89)                          # mov [rbp + disp], r64
+          emit_modrm_rbp_disp(reg & 7, disp)
+        end
+      end
+
+      # Puts the caller's values back, the exact inverse of
+      # #emit_save_promoted_registers (8B reads memory into the register). Every
+      # :ret emits this immediately before its "leave", and a :ret is the only
+      # way out of a function, so no path returns with a callee-saved register
+      # holding this function's value.
+      def emit_restore_promoted_registers
+        @promoted_saves.each do |reg, disp|
+          emit(0x48 | (reg >= 8 ? 0x04 : 0)) # REX.W (+ REX.R for r12..r15)
+          emit(0x8B)                          # mov r64, [rbp + disp]
+          emit_modrm_rbp_disp(reg & 7, disp)
+        end
+      end
+
       # Brings each incoming argument into its parameter slot (the first
       # `param_count` vregs, one per param_kinds entry) so it reads back like any
       # other vreg. The generator has already fixed where each parameter arrives:
@@ -269,11 +395,21 @@ module Rubycc
           load_reg(EAX, inst.a)
           store_reg(EAX, inst.dst)
         when :add
-          emit_binary(inst.dst, inst.a, inst.b, [0x01, 0xC8], inst.size) # add eax, ecx
+          # 03 /r is "add r32, r/m32" — the same addition as 01 /r's "add
+          # r/m32, r32" with the operands the other way round, which is what
+          # lets the second one be a slot. Every entry below pairs the two
+          # directions the same way (the group's low opcode plus two).
+          emit_binary(inst.dst, inst.a, inst.b, [0x01, 0xC8], inst.size,
+                      memory_bytes: [0x03], commutative: true)
+        when :scaled_add
+          emit_scaled_add(inst.dst, inst.a, inst.b, inst.size)
         when :sub
-          emit_binary(inst.dst, inst.a, inst.b, [0x29, 0xC8], inst.size) # sub eax, ecx
+          emit_binary(inst.dst, inst.a, inst.b, [0x29, 0xC8], inst.size, memory_bytes: [0x2B])
         when :mul
-          emit_binary(inst.dst, inst.a, inst.b, [0x0F, 0xAF, 0xC1], inst.size) # imul eax, ecx
+          # imul (0F AF /r) is already a "r32, r/m32" form, so its register and
+          # memory encodings share the same opcode.
+          emit_binary(inst.dst, inst.a, inst.b, [0x0F, 0xAF, 0xC1], inst.size,
+                      memory_bytes: [0x0F, 0xAF], commutative: true)
         when :mulhi
           emit_mulhi(inst.dst, inst.a, inst.b) # high 64 bits of an unsigned 64x64 product
         when :div
@@ -285,11 +421,14 @@ module Rubycc
         when :umod
           emit_udivmod(inst.dst, inst.a, inst.b, EDX, inst.size) # remainder in edx
         when :and
-          emit_binary(inst.dst, inst.a, inst.b, [0x21, 0xC8], inst.size) # and eax, ecx
+          emit_binary(inst.dst, inst.a, inst.b, [0x21, 0xC8], inst.size,
+                      memory_bytes: [0x23], commutative: true)
         when :or
-          emit_binary(inst.dst, inst.a, inst.b, [0x09, 0xC8], inst.size)  # or eax, ecx
+          emit_binary(inst.dst, inst.a, inst.b, [0x09, 0xC8], inst.size,
+                      memory_bytes: [0x0B], commutative: true)
         when :xor
-          emit_binary(inst.dst, inst.a, inst.b, [0x31, 0xC8], inst.size)  # xor eax, ecx
+          emit_binary(inst.dst, inst.a, inst.b, [0x31, 0xC8], inst.size,
+                      memory_bytes: [0x33], commutative: true)
         when :shl
           # shl eax, cl (D3 /4): the count comes from cl, which #emit_binary
           # loads into ecx alongside the value in eax. A size-8 operand takes a
@@ -306,7 +445,8 @@ module Rubycc
           # unsigned counterpart of :sar.
           emit_binary(inst.dst, inst.a, inst.b, [0xD3, 0xE8], inst.size)
         when :eq, :ne, :lt, :le, :gt, :ge, :ult, :ule, :ugt, :uge
-          emit_comparison(inst.dst, inst.a, inst.b, SETCC_OPCODES.fetch(inst.op), inst.size)
+          emit_comparison(inst.dst, inst.a, inst.b, SETCC_OPCODES.fetch(inst.op), inst.size,
+                          commutative: inst.op == :eq || inst.op == :ne)
         when :fadd
           emit_float_binary(inst.dst, inst.a, inst.b, inst.size, 0x58) # addss/addsd
         when :fsub
@@ -339,6 +479,11 @@ module Rubycc
         when :got_addr
           emit_got_addr(inst.dst, inst.a)
         when :label
+          # The one place a register's meaning changes without an instruction
+          # being emitted: control may arrive here from a branch whose registers
+          # hold something else entirely, so nothing may be assumed resident
+          # past a label (see SlotResidency).
+          forget_slot_residency
           @labels[inst.a] = @code.bytesize
         when :jump
           emit_jump(inst.a)
@@ -368,6 +513,8 @@ module Rubycc
           emit_alloca(inst.dst, inst.a)
         when :bit_scan
           emit_bit_scan(inst.dst, inst.a, inst.b, inst.size)
+        when :popcount
+          emit_popcount(inst.dst, inst.a, inst.size)
         when :atomic_load
           emit_atomic_load(inst.dst, inst.a, inst.size)
         when :atomic_store
@@ -591,6 +738,9 @@ module Rubycc
         else
           load_reg(EAX, value_vreg)
         end
+        # After the value is loaded, since it may itself come out of a promoted
+        # register, and before "leave" while rbp still addresses the frame.
+        emit_restore_promoted_registers
         emit(0xC9)                          # leave
         emit(0xC3)                          # ret
       end
@@ -683,39 +833,53 @@ module Rubycc
       end
 
       # :sext — sign-extend a's low `size` bytes to the full 64-bit register.
-      # size 4 is a movsxd rax, eax (the classic int -> long widening, so
-      # pointer-offset scaling sees a correct, possibly negative index); size 2
-      # a movsx rax, ax and size 1 a movsx rax, al, which re-derive a signed
-      # short/char value from just its stored low bytes. The 64-bit store writes
-      # the widened value back.
+      # size 4 is a movsxd (the classic int -> long widening, so pointer-offset
+      # scaling sees a correct, possibly negative index); size 2 a movsx r64,
+      # r/m16 and size 1 a movsx r64, r/m8, which re-derive a signed
+      # short/char value from just its stored low bytes.
+      #
+      # Both halves of the instruction are free of a fixed register: the source
+      # is an r/m field, so a promoted operand is read in place, and the
+      # destination a reg field, so a promoted result is written in place
+      # (#widen_in_registers).
       def emit_sext(dst, src_vreg, size)
-        load_reg(EAX, src_vreg)         # rax = value
         case size
-        when 1
-          emit(0x48, 0x0F, 0xBE, 0xC0)  # REX.W movsx rax, al
-        when 2
-          emit(0x48, 0x0F, 0xBF, 0xC0)  # REX.W movsx rax, ax
-        else # 4
-          emit(0x48, 0x63, 0xC0)        # REX.W movsxd rax, eax
+        when 1 then widen_in_registers(dst, src_vreg, [0x0F, 0xBE], rex_w: true) # movsx r64, r/m8
+        when 2 then widen_in_registers(dst, src_vreg, [0x0F, 0xBF], rex_w: true) # movsx r64, r/m16
+        else        widen_in_registers(dst, src_vreg, [0x63], rex_w: true)       # movsxd r64, r/m32
         end
-        store_reg(EAX, dst)
       end
 
       # :zext — zero-extend a's low `size` bytes to the full 64-bit register.
-      # size 1/2 are a movzx eax, al / movzx eax, ax; size 4 a plain mov eax,
-      # eax. Each writes eax, which x86-64 defines to clear the upper 32 bits of
-      # rax, so all three leave a clean 64-bit zero-extended value.
+      # size 1/2 are a movzx r32, r/m8 / movzx r32, r/m16; size 4 a plain 32-bit
+      # mov. Each writes a 32-bit register, which x86-64 defines to clear the
+      # upper 32 bits of its 64-bit whole, so all three leave a clean
+      # zero-extended value.
       def emit_zext(dst, src_vreg, size)
-        load_reg(EAX, src_vreg)         # rax = value
         case size
-        when 1
-          emit(0x0F, 0xB6, 0xC0)        # movzx eax, al
-        when 2
-          emit(0x0F, 0xB7, 0xC0)        # movzx eax, ax
-        else # 4
-          emit(0x89, 0xC0)              # mov eax, eax
+        when 1 then widen_in_registers(dst, src_vreg, [0x0F, 0xB6]) # movzx r32, r/m8
+        when 2 then widen_in_registers(dst, src_vreg, [0x0F, 0xB7]) # movzx r32, r/m16
+        else        widen_in_registers(dst, src_vreg, [0x8B])       # mov r32, r/m32
         end
-        store_reg(EAX, dst)
+      end
+
+      # The shared shape of :sext and :zext: one "reg, r/m" instruction from
+      # wherever the source lives to wherever the destination lives. A promoted
+      # source is the r/m operand outright; any other source goes through eax
+      # first, as before, since a slot holds eight bytes and these opcodes'
+      # 1/2-byte forms would read the wrong count of them. A promoted
+      # destination is the reg field, so no move follows either.
+      def widen_in_registers(dst, src_vreg, opcode_bytes, rex_w: false)
+        refresh_slot_residency
+        source = @promoted[src_vreg]
+        unless source
+          load_reg(EAX, src_vreg)       # rax = value
+          source = EAX
+        end
+        target = result_register(dst)
+        emit_register_rm(opcode_bytes, source, reg: target, rex_w: rex_w)
+        # One "reg, r/m" instruction: its reg field is the only thing written.
+        store_result(target, dst, only_wrote: target)
       end
 
       # :bit_scan — count the zero bits of a's value (__builtin_ctz/clz). The
@@ -740,6 +904,141 @@ module Rubycc
         store_reg(EAX, dst)
       end
 
+      # The bit-field patterns :popcount folds over, in their 8-byte form; a
+      # 4-byte count uses the low half of each. The first three keep the partial
+      # counts of one width from spilling into their neighbours, and the last is
+      # the one-per-byte multiplier that sums the eight byte counts at once.
+      POPCOUNT_PAIR_MASK = 0x5555_5555_5555_5555
+      POPCOUNT_NIBBLE_MASK = 0x3333_3333_3333_3333
+      POPCOUNT_BYTE_MASK = 0x0F0F_0F0F_0F0F_0F0F
+      POPCOUNT_BYTE_ONES = 0x0101_0101_0101_0101
+
+      # :popcount — how many bits of a's value are set (__builtin_popcount and
+      # its "l"/"ll" forms). x86-64 does have an instruction for exactly this,
+      # `popcnt`, but it arrived with SSE4.2 and is not part of the baseline
+      # architecture rubycc emits for: using it would make the output fault on
+      # an older CPU, and make it depend on the CPU that happened to build it.
+      # The count is expanded instead, in the same shape the aarch64 backend
+      # uses (whose `cnt` is an extension too — an AdvSIMD one, reached only
+      # through a vector register).
+      #
+      # The expansion is a divide-and-conquer sum over the register's own bits
+      # ("SWAR"): a register is read as many independent counts side by side,
+      # which are added pairwise until one of them spans it. Start from 2-bit
+      # fields, where the count is already there to be read — a field holding
+      # hi:lo has hi + lo bits set — and each step halves the number of fields
+      # while doubling their width: 32 counts become 16, then 8, 4, 2 and 1 in a
+      # 64-bit register, half that in a 32-bit one. What every step needs is a
+      # shift to bring one field's partner onto it, an add, and enough masking
+      # that no field's sum reaches its neighbour. Three steps are written out
+      # below and the remaining ones — three of them at 64 bits, two at 32 —
+      # are what the closing multiply performs in a single instruction:
+      #
+      #   pairs    x -= (x >> 1) & 0x5555...   each 2-bit field v = 2*hi + lo
+      #                                        becomes v - hi = hi + lo. The
+      #                                        subtraction never borrows across
+      #                                        a field (the result is at most
+      #                                        the field's own value), so the
+      #                                        one mask on the shift is all it
+      #                                        takes
+      #   nibbles  x = (x & 0x3333...) + ((x >> 2) & 0x3333...)
+      #                                        two counts of at most 2 sum to 4,
+      #                                        which no longer fits a 2-bit
+      #                                        field: both sides are masked
+      #                                        before adding, into the 4-bit
+      #                                        fields that do hold it
+      #   bytes    x = (x + (x >> 4)) & 0x0F0F...
+      #                                        two counts of at most 4 sum to 8,
+      #                                        which still fits a nibble, so the
+      #                                        add can run unmasked and one mask
+      #                                        afterwards clears the odd nibbles
+      #                                        it dirtied
+      #   total    (x * 0x0101...) >> (bits - 8)
+      #                                        every byte now holds its own
+      #                                        count. Multiplying by one-per-byte
+      #                                        adds each byte into every byte
+      #                                        above it, so the top byte receives
+      #                                        all of them; the column sums are
+      #                                        at most 64 and never carry out of
+      #                                        a byte, and the shift brings that
+      #                                        top byte down
+      #
+      # rcx carries the running value, rax the copy each stage shifts and adds
+      # back in (and, from the third stage on, the result), and rdx a mask —
+      # which only the 8-byte form needs, x86-64 having no 64-bit immediate for
+      # a logical op. The 4-byte form is the same sequence in 32-bit registers,
+      # whose writes zero the upper half, so the indeterminate high bits of a
+      # slot that was moved 64 bits at a time never enter the count.
+      def emit_popcount(dst, src_vreg, size)
+        load_reg(ECX, src_vreg)                            # rcx = value to count
+        emit_popcount_reg(0x89, EAX, ECX, size)            # mov rax, rcx
+        emit_popcount_shr(EAX, 1, size)                    # rax = x >> 1
+        emit_popcount_mask([EAX], POPCOUNT_PAIR_MASK, size)
+        emit_popcount_reg(0x29, ECX, EAX, size)            # sub rcx, rax  (pair counts)
+        emit_popcount_reg(0x89, EAX, ECX, size)            # mov rax, rcx
+        emit_popcount_shr(ECX, 2, size)                    # rcx = x >> 2
+        emit_popcount_mask([EAX, ECX], POPCOUNT_NIBBLE_MASK, size)
+        emit_popcount_reg(0x01, ECX, EAX, size)            # add rcx, rax  (nibble counts)
+        emit_popcount_reg(0x89, EAX, ECX, size)            # mov rax, rcx
+        emit_popcount_shr(EAX, 4, size)                    # rax = x >> 4
+        emit_popcount_reg(0x01, EAX, ECX, size)            # add rax, rcx
+        emit_popcount_mask([EAX], POPCOUNT_BYTE_MASK, size) # byte counts
+        emit_popcount_byte_sum(size)                       # rax = sum of the bytes
+        store_reg(EAX, dst)
+      end
+
+      # One two-register instruction of the expansion, in the "r/m, reg"
+      # direction: `opcode` applies `reg` to `rm`. An 8-byte count takes a REX.W
+      # prefix and a 4-byte one no prefix at all — every register here is one of
+      # rax/rcx/rdx, so no REX bit is needed to name them.
+      def emit_popcount_reg(opcode, rm, reg, size)
+        emit(0x48) if size == 8                            # REX.W: the 64-bit form
+        emit(opcode, 0xC0 | (reg << 3) | rm)
+      end
+
+      # `shr rm, count` (C1 /5 ib) at the count's width.
+      def emit_popcount_shr(rm, count, size)
+        emit(0x48) if size == 8                            # REX.W: a 64-bit shift
+        emit(0xC1, 0xE8 | rm, count)
+      end
+
+      # Masks each of `regs` with one of the expansion's field patterns. A
+      # 4-byte count names it as an immediate ("and r/m32, imm32", 81 /4 id); an
+      # 8-byte one cannot, x86-64's logical ops taking at most a sign-extended
+      # imm32, so the mask is materialized in rdx once (movabs) and applied from
+      # there ("and r/m64, r64", 21 /r) — which is why the registers to mask
+      # arrive together rather than one call at a time: the nibble stage masks
+      # two of them with the same pattern.
+      def emit_popcount_mask(regs, mask, size)
+        if size == 8
+          emit(0x48, 0xBA)                                 # movabs rdx, imm64
+          emit_bytes([mask].pack("Q<"))
+          regs.each { |reg| emit_popcount_reg(0x21, reg, EDX, size) } # and reg, rdx
+        else
+          regs.each do |reg|
+            emit(0x81, 0xE0 | reg)                         # and r/m32, imm32
+            emit_bytes([mask & 0xFFFFFFFF].pack("L<"))
+          end
+        end
+      end
+
+      # The expansion's last stage: rax holds one count per byte, and the
+      # multiply by one-per-byte gathers them into the top byte, which the shift
+      # brings down. The 4-byte form takes its multiplier as an immediate
+      # ("imul r32, r/m32, imm32", 69 /r id); the 8-byte one has no imm64 form
+      # either, so the multiplier goes through rdx ("imul r64, r/m64", 0F AF /r).
+      def emit_popcount_byte_sum(size)
+        if size == 8
+          emit(0x48, 0xBA)                                 # movabs rdx, 0x0101010101010101
+          emit_bytes([POPCOUNT_BYTE_ONES].pack("Q<"))
+          emit(0x48, 0x0F, 0xAF, 0xC2)                     # imul rax, rdx
+        else
+          emit(0x69, 0xC0)                                 # imul eax, eax, imm32
+          emit_bytes([POPCOUNT_BYTE_ONES & 0xFFFFFFFF].pack("L<"))
+        end
+        emit_popcount_shr(EAX, size * 8 - 8, size)         # rax = the top byte
+      end
+
       # --- atomics -----------------------------------------------------------
       #
       # The four atomic ops all lower at sequential consistency, the strongest
@@ -782,8 +1081,7 @@ module Rubycc
       # implicitly locked, is also the full barrier a seq_cst store needs; a mov
       # followed by an mfence would be equivalent and longer.
       def emit_atomic_store(ptr_vreg, value_vreg, size)
-        load_reg(EAX, ptr_vreg)         # rax = destination address
-        load_reg(ECX, value_vreg)       # rcx = value
+        load_binary_operands(ptr_vreg, value_vreg) # rax = destination address, rcx = value
         emit(0x48) if size == 8         # REX.W
         emit(0x87, 0x08)                # xchg [rax], ecx/rcx
       end
@@ -805,8 +1103,7 @@ module Rubycc
       def emit_atomic_rmw(dst, ptr_vreg, value_vreg, kind, size)
         return emit_atomic_or_fetch(dst, ptr_vreg, value_vreg, size) if kind == :or_fetch
 
-        load_reg(EAX, ptr_vreg)         # rax = address
-        load_reg(ECX, value_vreg)       # rcx = operand
+        load_binary_operands(ptr_vreg, value_vreg) # rax = address, rcx = operand
         negate = kind == :fetch_sub || kind == :sub_fetch
         keep = kind == :add_fetch || kind == :sub_fetch
         if negate
@@ -906,24 +1203,21 @@ module Rubycc
         store_reg(EAX, dst)
       end
 
-      # "*p" read, sign-extending: load the pointer from its slot into rax, then
-      # read through it. An 8-byte load moves a full pointer/long (mov rax,
-      # [rax]); a 4-byte load reads an int (mov eax, [rax]), whose upper bits the
-      # write to eax zeroes; a 2/1-byte load reads a short/char and sign-extends
-      # it to the register (movsx), so the slot holds a promoted signed value.
+      # "*p" read, sign-extending: load the pointer into rax, then read through
+      # it. An 8-byte load moves a full pointer/long (mov r64, [rax]); a 4-byte
+      # load reads an int (mov r32, [rax]), whose upper bits the 32-bit write
+      # zeroes; a 2/1-byte load reads a short/char and sign-extends it to the
+      # register (movsx), so the slot holds a promoted signed value.
+      #
+      # The destination is a reg field in every one of the four, so a promoted
+      # one is written directly (#emit_deref).
       def emit_load(dst, ptr_vreg, size)
-        load_reg(EAX, ptr_vreg)         # rax = pointer value
         case size
-        when 8
-          emit(0x48, 0x8B, 0x00)        # mov rax, [rax]
-        when 2
-          emit(0x0F, 0xBF, 0x00)        # movsx eax, word [rax]
-        when 1
-          emit(0x0F, 0xBE, 0x00)        # movsx eax, byte [rax]
-        else # 4
-          emit(0x8B, 0x00)              # mov eax, [rax]
+        when 8 then emit_deref(dst, ptr_vreg, [0x8B], rex_w: true)  # mov r64, [rax]
+        when 2 then emit_deref(dst, ptr_vreg, [0x0F, 0xBF])         # movsx r32, word [rax]
+        when 1 then emit_deref(dst, ptr_vreg, [0x0F, 0xBE])         # movsx r32, byte [rax]
+        else        emit_deref(dst, ptr_vreg, [0x8B])               # mov r32, [rax]
         end
-        store_reg(EAX, dst)
       end
 
       # "*p" read, zero-extending: the unsigned counterpart of #emit_load for an
@@ -931,18 +1225,28 @@ module Rubycc
       # forms are identical to a signed load (a plain mov already zero-extends a
       # 32-bit read, and an 8-byte value has no spare bits).
       def emit_uload(dst, ptr_vreg, size)
-        load_reg(EAX, ptr_vreg)         # rax = pointer value
         case size
-        when 8
-          emit(0x48, 0x8B, 0x00)        # mov rax, [rax]
-        when 2
-          emit(0x0F, 0xB7, 0x00)        # movzx eax, word [rax]
-        when 1
-          emit(0x0F, 0xB6, 0x00)        # movzx eax, byte [rax]
-        else # 4
-          emit(0x8B, 0x00)              # mov eax, [rax]
+        when 8 then emit_deref(dst, ptr_vreg, [0x8B], rex_w: true)  # mov r64, [rax]
+        when 2 then emit_deref(dst, ptr_vreg, [0x0F, 0xB7])         # movzx r32, word [rax]
+        when 1 then emit_deref(dst, ptr_vreg, [0x0F, 0xB6])         # movzx r32, byte [rax]
+        else        emit_deref(dst, ptr_vreg, [0x8B])               # mov r32, [rax]
         end
-        store_reg(EAX, dst)
+      end
+
+      # The shared shape of :load and :uload: the pointer into rax and one read
+      # through [rax] into wherever the destination lives. The ModR/M is mod =
+      # 00 with rm = 000, the plain "[rax]" form — no SIB byte and no
+      # displacement — so only the reg field varies, and REX.R is all a promoted
+      # destination costs.
+      def emit_deref(dst, ptr_vreg, opcode_bytes, rex_w: false)
+        load_reg(EAX, ptr_vreg)         # rax = pointer value
+        target = result_register(dst)
+        emit_rex(rex_w: rex_w, reg: target)
+        opcode_bytes.each { |byte| emit(byte) }
+        emit((target & 7) << 3)         # mod=00, rm=000: [rax]
+        # The read through [rax] writes its reg field and nothing else; rax
+        # itself was written by the load above, which accounted for itself.
+        store_result(target, dst, only_wrote: target)
       end
 
       # "*p = v": rax holds the destination address, rcx the value, then rcx is
@@ -952,8 +1256,7 @@ module Rubycc
       # a 1-byte store writes just the low byte (mov [rax], cl). The narrower
       # writes are exactly the truncation a narrow lvalue needs.
       def emit_store(ptr_vreg, value_vreg, size)
-        load_reg(EAX, ptr_vreg)         # rax = destination address
-        load_reg(ECX, value_vreg)       # rcx = value
+        load_binary_operands(ptr_vreg, value_vreg) # rax = destination address, rcx = value
         case size
         when 8
           emit(0x48, 0x89, 0x08)        # mov [rax], rcx
@@ -1038,33 +1341,83 @@ module Rubycc
         store_reg(RSP, dst)                 # dst = rsp (block base address)
       end
 
-      # A size of 8 compares full 64-bit pointer values (REX.W cmp rax, rcx);
-      # otherwise the 32-bit int compare is used. The signed setcc still suits
-      # pointer ordering here, since stack addresses stay within the positive
-      # half of the 64-bit range.
-      def emit_comparison(dst, a, b, setcc_opcode, size = nil)
-        load_reg(EAX, a)
-        load_reg(ECX, b)
-        emit(0x48) if size == 8         # REX.W widens the following cmp
-        emit(0x39, 0xC8)                # cmp eax, ecx  (rax, rcx when REX.W)
+      # A size of 8 compares full 64-bit pointer values (REX.W); otherwise the
+      # 32-bit int compare is used. The signed setcc still suits pointer
+      # ordering here, since stack addresses stay within the positive half of
+      # the 64-bit range. The second operand is named where it lives (3B /r,
+      # "cmp r32, r/m32") wherever it can be, as in #emit_binary.
+      #
+      # `cmp` reads both of its operands and writes neither, which the first one
+      # takes advantage of too: a promoted `a` is named in the reg field
+      # directly rather than moved into eax to be looked at. The result still
+      # goes through eax, `setcc` writing al and nothing else — one of the fixed
+      # registers this file does not try to talk out of its choice.
+      #
+      # An equality test is the one comparison whose operands may be exchanged
+      # without changing its setcc: "a == b" and "b == a" leave the same ZF,
+      # while every ordering test would need its condition mirrored as well.
+      def emit_comparison(dst, a, b, setcc_opcode, size = nil, commutative: false)
+        refresh_slot_residency
+        a, b = b, a if commutative && prefer_swapped_operands?(a, b)
+        if rm_operand?(b)
+          left = @promoted[a]
+          unless left
+            load_reg(EAX, a)
+            left = EAX
+          end
+          emit_vreg_rm([0x3B], b, reg: left, rex_w: size == 8) # cmp r32, r/m32
+        else
+          load_binary_operands(a, b, commutative: commutative)
+          emit(0x48) if size == 8       # REX.W widens the following cmp
+          emit(0x39, 0xC8)              # cmp eax, ecx  (rax, rcx when REX.W)
+        end
         emit(0x0F, setcc_opcode, 0xC0)  # setcc al
         emit(0x0F, 0xB6, 0xC0)          # movzx eax, al
         store_reg(EAX, dst)
       end
 
-      # A floating binary op (:fadd/:fsub/:fmul/:fdiv). The operands are loaded
-      # into xmm0/xmm1 from their slots, combined in place, and stored back. The
-      # mandatory prefix selects the scalar-single (F3, size 4) or scalar-double
-      # (F2, size 8) form of the shared 0F <opcode> encoding.
+      # A floating binary op (:fadd/:fsub/:fmul/:fdiv). The first operand is
+      # loaded into xmm0, the second read straight out of its slot, and the
+      # result stored back. The mandatory prefix selects the scalar-single (F3,
+      # size 4) or scalar-double (F2, size 8) form of the shared 0F <opcode>
+      # encoding, and all four take an "xmm, xmm/m" operand pair — so the memory
+      # form is the same instruction with a different ModR/M, one shorter than
+      # staging the operand through xmm1 would be.
+      #
+      # The operands are never exchanged the way a commutative integer op's are:
+      # SSE arithmetic takes a NaN result's payload from a particular source
+      # operand, so the two orders are not interchangeable even where the
+      # arithmetic itself is.
       def emit_float_binary(dst, a, b, size, opcode)
-        load_xmm(XMM0, a, size)
-        load_xmm(XMM1, b, size)
+        load_float_operands(a, b, size)
         emit(size == 8 ? 0xF2 : 0xF3)
         emit(0x0F, opcode)
-        emit(modrm_reg(XMM0, XMM1))     # op xmm0, xmm1
+        emit_float_rm(b)
         store_xmm(XMM0, dst, size)
       end
 
+      # Stages a floating instruction's operands: the first into xmm0, and the
+      # second into xmm1 only when its slot was never written (a transient left
+      # in a register, which no memory operand can name). The rescue happens
+      # *before* xmm0 is filled, because the value being rescued is the one
+      # sitting in xmm0 — this is the vector-file version of the ordering rule
+      # #load_binary_operands follows.
+      def load_float_operands(a, b, size)
+        refresh_slot_residency
+        load_xmm(XMM1, b, size) unless slot_written?(b)
+        load_xmm(XMM0, a, size)
+      end
+
+      # The r/m half of a floating instruction whose reg half is xmm0: b's slot
+      # when it was written, xmm1 when #load_float_operands had to stage it.
+      def emit_float_rm(b)
+        if slot_written?(b)
+          emit_modrm_rbp_disp(XMM0, slot_disp(b))
+        else
+          emit(modrm_reg(XMM0, XMM1))
+        end
+      end
+
       # A floating comparison (:feq..:fge), materialized into eax as an int 0/1
       # like an integer comparison but through ucomiss/ucomisd, whose result is
       # read from the flags with NaN awareness. The ordering ops reduce to a
@@ -1109,15 +1462,15 @@ module Rubycc
         store_reg(EAX, dst)
       end
 
-      # ucomiss xmm0, xmm1 (size 4) / ucomisd (size 8, a 66 prefix), the ordered
-      # scalar compare that sets ZF/PF/CF for the setcc that follows. The two
-      # operands are loaded into xmm0/xmm1 from their slots first.
+      # ucomiss xmm0, xmm/m32 (size 4) / ucomisd (size 8, a 66 prefix), the
+      # ordered scalar compare that sets ZF/PF/CF for the setcc that follows.
+      # The first operand is loaded into xmm0 and the second named where it
+      # lives, exactly as #emit_float_binary does.
       def emit_ucomis(a, b, size)
-        load_xmm(XMM0, a, size)
-        load_xmm(XMM1, b, size)
+        load_float_operands(a, b, size)
         emit(0x66) if size == 8         # ucomisd operand-size prefix
         emit(0x0F, 0x2E)                # ucomiss/ucomisd
-        emit(modrm_reg(XMM0, XMM1))
+        emit_float_rm(b)
       end
 
       # :itof — cvtsi2ss/cvtsi2sd converts a signed integer in a GP register to
@@ -1177,18 +1530,41 @@ module Rubycc
       # single/double form; the rbp-relative ModR/M reuses the integer helper,
       # the xmm number sitting in its reg field (0..7 for the scratch pair and the
       # eight argument registers alike, all within the 3-bit field, so no REX.R).
+      #
+      # As with #load_reg, the move disappears when `xmm` already holds this
+      # slot at this width and becomes a register-to-register move when another
+      # vector register does. The register form is `movaps` (0F 28 /r), which
+      # copies all 128 bits: that is exactly what a reload would have produced,
+      # since a movss/movsd *from memory* zeroes the bits above the value it
+      # loads, so the source register's upper half is already what the
+      # destination's would be.
       def load_xmm(xmm, vreg, size)
-        emit(size == 8 ? 0xF2 : 0xF3)
-        emit(0x0F, 0x10)
-        emit_modrm_rbp_disp(xmm, slot_disp(vreg))
+        refresh_slot_residency
+        return if slot_resident_in_vector?(xmm, vreg, size)
+
+        source = vector_register_holding_slot(vreg, size)
+        if source
+          emit(0x0F, 0x28, modrm_reg(xmm, source)) # movaps xmm, xmm_source
+        else
+          emit(size == 8 ? 0xF2 : 0xF3)
+          emit(0x0F, 0x10)
+          emit_modrm_rbp_disp(xmm, slot_disp(vreg))
+        end
+        note_slot_loaded_to_vector(xmm, vreg, size)
       end
 
       # movss/movsd [rbp + disp], xmm: stores an xmm register into a slot, the
       # counterpart of #load_xmm (opcode 0x11 writes memory from the register).
       def store_xmm(xmm, vreg, size)
+        refresh_slot_residency
+        if @transient[vreg]
+          note_slot_loaded_to_vector(xmm, vreg, size)
+          return
+        end
         emit(size == 8 ? 0xF2 : 0xF3)
         emit(0x0F, 0x11)
         emit_modrm_rbp_disp(xmm, slot_disp(vreg))
+        note_slot_stored_from_vector(xmm, vreg, size)
       end
 
       # Emits "jmp rel32" with a zero placeholder and records a fixup so the
@@ -1198,9 +1574,20 @@ module Rubycc
         record_fixup(label_id)
       end
 
+      # The condition is tested where it lives unless it is already in eax: a
+      # "cmp dword [rbp+disp], 0" — or "cmp r32, 0" for a promoted condition —
+      # reads the same low four bytes "test eax, eax" would have, and saves the
+      # load. (The 83 /7 form takes a sign-extended imm8, which is all a
+      # comparison against zero needs.)
       def emit_jump_if_zero(cond, label_id)
-        load_reg(EAX, cond)
-        emit(0x85, 0xC0)                # test eax, eax
+        refresh_slot_residency
+        if rm_operand?(cond) && !slot_resident_in?(EAX, cond)
+          emit_vreg_rm([0x83], cond, reg: 7)
+          emit(0x00)                    # cmp r/m32, 0
+        else
+          load_reg(EAX, cond)
+          emit(0x85, 0xC0)              # test eax, eax
+        end
         emit(0x0F, 0x84)                # je rel32
         record_fixup(label_id)
       end
@@ -1227,74 +1614,345 @@ module Rubycc
       # A size of 8 materializes a full 64-bit immediate (movabs rax, imm64),
       # needed for a `long`/`unsigned long` constant that does not fit — or
       # would not sign-extend correctly — in 32 bits. Otherwise a 32-bit
-      # "mov eax, imm32" suffices: it fills the low 32 bits (the whole value of
-      # a 4-byte-or-narrower type) and zeroes the upper half of rax.
+      # "mov r32, imm32" suffices: it fills the low 32 bits (the whole value of
+      # a 4-byte-or-narrower type) and zeroes the register's upper half.
+      #
+      # "mov r64/r32, imm" (B8+rd) names its destination in the opcode's low
+      # three bits, extended by REX.B, so a promoted destination is written
+      # directly and no move follows (#result_register).
       def emit_const(dst, value, size = nil)
+        refresh_slot_residency
+        target = result_register(dst)
         if size == 8
-          emit(0x48, 0xB8)                                    # movabs rax, imm64
+          emit_rex(rex_w: true, rm: target)                   # REX.W (+ REX.B)
+          emit(0xB8 | (target & 7))                           # movabs r64, imm64
           emit_bytes([value & 0xFFFFFFFFFFFFFFFF].pack("Q<"))
         else
-          emit(0xB8)                                          # mov eax, imm32
+          emit_rex(rm: target)                                # REX.B for r12..r15
+          emit(0xB8 | (target & 7))                           # mov r32, imm32
           emit_bytes([value & 0xFFFFFFFF].pack("L<"))
         end
-        store_reg(EAX, dst)
+        # Either form is one "mov r, imm" writing the opcode's own register.
+        store_result(target, dst, only_wrote: target)
       end
 
       # A size of 8 prefixes REX.W so the operation runs on the full 64-bit
-      # rax/rcx (pointer arithmetic and index scaling); otherwise it stays a
-      # 32-bit int operation. The opcode bytes are identical either way.
-      def emit_binary(dst, a, b, opcode_bytes, size = nil)
-        load_reg(EAX, a)
-        load_reg(ECX, b)
-        emit(0x48) if size == 8
+      # rax (pointer arithmetic and index scaling); otherwise it stays a 32-bit
+      # int operation. The opcode bytes are identical either way.
+      #
+      # `memory_bytes`, when the op has such a form, is the opcode of its
+      # "register, r/m" direction, which #emit_vreg_rm then points at b's own
+      # slot or promoted register — so only `a` is ever loaded and the whole
+      # instruction costs one load plus itself. `opcode_bytes` is the fallback
+      # "eax, ecx" encoding, used by the shifts, whose count has to reach cl and
+      # cannot be a memory operand at all. `commutative` lets the two be
+      # exchanged, which is worth doing when it makes the one remaining load
+      # free — or when it puts a promoted destination on the left, where the
+      # result can be accumulated in place (#emit_promoted_binary).
+      def emit_binary(dst, a, b, opcode_bytes, size = nil, memory_bytes: nil, commutative: false)
+        refresh_slot_residency
+        a, b = b, a if commutative && swap_binary_operands?(dst, a, b)
+        if memory_bytes && @promoted[dst] && (dst == a || dst != b)
+          emit_promoted_binary(dst, a, b, memory_bytes, size)
+        elsif memory_bytes && rm_operand?(b)
+          load_reg(EAX, a)
+          emit_vreg_rm(memory_bytes, b, rex_w: size == 8)
+          store_reg(EAX, dst)
+        else
+          load_binary_operands(a, b, commutative: commutative)
+          emit(0x48) if size == 8
+          opcode_bytes.each { |byte| emit(byte) }
+          store_reg(EAX, dst)
+        end
+      end
+
+      # The "register, r/m" direction of a binary op with its reg field aimed
+      # straight at the callee-saved register `dst` was promoted into, so the
+      # result is written where it lives and no move follows.
+      #
+      # `a` is brought into that register first, a move that disappears when
+      # `dst` and `a` are the same value — the shape "i = i + 1" and "sum += x"
+      # take once the single-use copies are forwarded, and the reason this path
+      # exists. The caller has ruled out the one arrangement the move would
+      # break: dst == b with dst != a, whose b it would overwrite before the
+      # arithmetic could read it.
+      #
+      # Only the promoted register is written, and the residency table can name
+      # none of them (PROMOTION_REGISTERS is disjoint from the scratch,
+      # argument and return registers every entry is keyed by), so every
+      # residency in flight survives; #note_register_clobbered still drops the
+      # one the `load_reg` above may have recorded against the target itself.
+      def emit_promoted_binary(dst, a, b, memory_bytes, size)
+        target = @promoted[dst]
+        load_reg(target, a) unless dst == a
+        if rm_operand?(b)
+          emit_vreg_rm(memory_bytes, b, reg: target, rex_w: size == 8)
+        else
+          # A transient has neither slot nor register of its own; it is still
+          # in whichever scratch register its producer computed it in.
+          source = register_holding_slot(b)
+          unless source
+            load_reg(ECX, b)
+            source = ECX
+          end
+          emit_register_rm(memory_bytes, source, reg: target, rex_w: size == 8)
+        end
+        note_register_clobbered(target)
+      end
+
+      # Whether `vreg` may be named as an instruction's r/m operand, and so
+      # read without being loaded first. A value in a slot is named [rbp +
+      # disp] and a promoted one register-direct; only a transient is neither,
+      # its value never having reached a slot and its register being whichever
+      # one its producer happened to use.
+      def rm_operand?(vreg)
+        !@transient[vreg]
+      end
+
+      # Emits an instruction whose r/m operand is wherever `vreg` lives: its
+      # stack slot, or — for a promoted value, which has no slot at all — the
+      # callee-saved register it owns, named register-direct (mod = 11). `reg`
+      # is the ModR/M reg field, a register number in a two-operand form and an
+      # opcode extension in a group instruction. The caller must have asked
+      # #rm_operand? first.
+      def emit_vreg_rm(opcode_bytes, vreg, reg: EAX, rex_w: false)
+        promoted = @promoted[vreg]
+        if promoted
+          emit_register_rm(opcode_bytes, promoted, reg: reg, rex_w: rex_w)
+        else
+          emit_slot_rm(opcode_bytes, vreg, reg: reg, rex_w: rex_w)
+        end
+      end
+
+      # Emits an instruction whose r/m operand is `vreg`'s stack slot: the REX
+      # prefix the operand width and register numbers call for, the opcode
+      # bytes, then a ModR/M whose reg field is `reg` and whose r/m field
+      # addresses [rbp + slot].
+      #
+      # Reading an operand straight out of its slot is as correct as loading it
+      # into a register first and is one instruction shorter. The slot is where
+      # the spill-everything discipline keeps the value; nothing is emitted
+      # between the load of the first operand and this that could write memory;
+      # and the width the instruction reads (4 bytes, or 8 under REX.W) is
+      # exactly the width the register form's arithmetic would have used, so the
+      # value representation is honored unchanged — a narrow value's
+      # indeterminate high half is never looked at either way.
+      def emit_slot_rm(opcode_bytes, vreg, reg: EAX, rex_w: false)
+        emit_rex(rex_w: rex_w, reg: reg)
         opcode_bytes.each { |byte| emit(byte) }
-        store_reg(EAX, dst)
+        emit_modrm_rbp_disp(reg & 7, slot_disp(vreg))
+      end
+
+      # The register-direct counterpart of #emit_slot_rm: the same reg field
+      # against an rm field that names register `rm` (mod = 11) rather than a
+      # memory operand. This is how a promoted value is read in place. Two
+      # encoding traps a memory operand has do not exist here — an rm of 100
+      # names r12/rsp instead of introducing a SIB byte, and an rm of 101 names
+      # r13/rbp instead of meaning "no base" — so REX.B is all r12..r15 need.
+      def emit_register_rm(opcode_bytes, rm, reg: EAX, rex_w: false)
+        emit_rex(rex_w: rex_w, reg: reg, rm: rm)
+        opcode_bytes.each { |byte| emit(byte) }
+        emit(0xC0 | ((reg & 7) << 3) | (rm & 7))
+      end
+
+      # The REX prefix an instruction needs, emitted only when some field asks
+      # for one. W widens the operation to 64 bits; R extends the ModR/M reg
+      # field, X the SIB index field and B the ModR/M r/m (or SIB base, or
+      # opcode-embedded) register — each supplying the high bit of a register
+      # number 8..15 whose low three bits stay in the field itself.
+      def emit_rex(rex_w: false, reg: 0, rm: 0, index: 0)
+        rex = 0x40 | (rex_w ? 0x08 : 0) | (reg >= 8 ? 0x04 : 0) |
+              (index >= 8 ? 0x02 : 0) | (rm >= 8 ? 0x01 : 0)
+        emit(rex) unless rex == 0x40
+      end
+
+      # Where an instruction that computes a general-register result should put
+      # it: the callee-saved register `dst` was promoted into, so the value is
+      # written home directly, or eax for a value that lives in a slot.
+      def result_register(dst)
+        @promoted[dst] || EAX
+      end
+
+      # Completes such an instruction. A destination that lives in a slot still
+      # needs the store; a promoted `dst` *is* `reg`, so the value is already
+      # home and all that is left is telling the residency table what the bytes
+      # just emitted did to it.
+      #
+      # **The default answer is to throw the table away.** Bytes have been
+      # emitted since it was last known true — the caller's own instruction —
+      # and nothing here can see which registers they wrote. Keeping the table
+      # alive across them is #note_register_clobbered's exception, sound only
+      # when those bytes wrote `reg` and nothing else the table can name; that
+      # is a fact about one lowering's encoding, so the lowering is what states
+      # it, by passing `only_wrote: reg`. Anything that says nothing gets the
+      # safe reading, which costs an optimization and never correctness — and a
+      # lowering that later grows a second scratch write has to come back to its
+      # own claim rather than silently reviving a stale residency. All four
+      # callers here claim the fast path today; the aarch64 backend, whose
+      # #emit_alloca rounds its size in rax's counterpart and whose remainder
+      # takes a scratch for the quotient, has two that cannot.
+      #
+      # "Nothing else the table can name" means none of rax, rcx, rdx, rsi, rdi,
+      # r10, the argument registers or a promoted one — the registers #load_reg
+      # is ever asked to fill, and so the only ones an entry can be keyed by.
+      def store_result(reg, dst, only_wrote: nil)
+        unless @promoted[dst]
+          store_reg(reg, dst) # which discards a stale table itself, before anything else
+          return
+        end
+
+        if only_wrote == reg
+          note_register_clobbered(reg)
+        else
+          refresh_slot_residency
+        end
+      end
+
+      # :scaled_add — dst <- base + index * element_size, the address a
+      # subscript forms. `lea` (8D /r) computes an address rather than loading
+      # from it, and its SIB byte carries the scale as a two-bit shift, so the
+      # element-size multiply and the add to the base become one instruction
+      # that touches no flags. REX.W makes it a 64-bit computation, which is
+      # what a pointer needs.
+      #
+      # All three of the instruction's registers are free — base and index are
+      # named independently in the SIB byte and the destination in the ModR/M
+      # reg field — so a promoted value may be any of them and the whole address
+      # is formed without a move. That freedom is what brings the three encoding
+      # traps of the SIB byte into reach, none of which a pair of scratch
+      # registers could ever have hit (Intel SDM Vol. 2A, Tables 2-5 and 2-3;
+      # REX.B extends the base, REX.X the index and REX.R the destination):
+      #
+      #   * a SIB **base** field of 101 with mod = 00 does not name rbp/r13 but
+      #     "no base register, disp32 follows", so those two take mod = 01 with
+      #     a zero disp8 instead — one byte for the register the encoding's
+      #     shorter form has spent on something else;
+      #   * a SIB base field of 100 does name rsp/r12 (it is the *ModR/M* rm
+      #     field where 100 means "a SIB byte follows"), so r12 as a base needs
+      #     nothing beyond its REX.B;
+      #   * a SIB **index** field of 100 means "no index" — but only at REX.X =
+      #     0. With REX.X set it is r12, which is why r12 works as an index and
+      #     rsp, whose REX.X is 0 by construction, is the register that cannot.
+      def emit_scaled_add(dst, base_vreg, index_vreg, element_size)
+        base, index = address_operands(base_vreg, index_vreg)
+        target = result_register(dst)
+        emit_rex(rex_w: true, reg: target, index: index, rm: base)
+        emit(0x8D)                              # lea r64, [base + index*scale]
+        sib = (SIB_SCALES.fetch(element_size) << 6) | ((index & 7) << 3) | (base & 7)
+        if (base & 7) == 5
+          emit(0x44 | ((target & 7) << 3), sib, 0x00) # mod=01, rm=100 (SIB), disp8 = 0
+        else
+          emit(0x04 | ((target & 7) << 3), sib)       # mod=00, rm=100 (SIB follows)
+        end
+        # `lea` writes its reg field; base and index are read (#address_operands
+        # loaded them and accounted for that itself).
+        store_result(target, dst, only_wrote: target)
+      end
+
+      # Reports which register holds a scaled address's base and which its
+      # index. A promoted operand is used where it already is; anything else is
+      # loaded into a scratch register. Unlike an arithmetic opcode, `lea` names
+      # its base and index independently, so either assignment encodes — which
+      # also lets whichever operand is already resident keep the register it is
+      # in when neither is promoted.
+      def address_operands(base_vreg, index_vreg)
+        refresh_slot_residency
+        base = @promoted[base_vreg]
+        index = @promoted[index_vreg]
+        if base && index
+          [base, index]
+        elsif base
+          load_reg(EAX, index_vreg)
+          [base, EAX]
+        elsif index
+          load_reg(EAX, base_vreg)
+          [EAX, index]
+        elsif slot_resident_in?(EAX, index_vreg) && !slot_resident_in?(EAX, base_vreg)
+          load_reg(ECX, base_vreg)
+          load_reg(EAX, index_vreg)
+          [ECX, EAX]
+        else
+          load_reg(EAX, base_vreg)
+          load_reg(ECX, index_vreg)
+          [EAX, ECX]
+        end
       end
 
       # :mulhi — the unsigned high 64 bits of a 64x64 product, the piece a
       # synthesized __int128 multiply needs beyond the low 64 that :mul gives.
-      # `mul rcx` (REX.W F7 /4) multiplies rax by rcx into rdx:rax; the high half
-      # lands in rdx, which is stored to the destination. The one-operand `mul`
-      # is the unsigned multiply, so this is the unsigned high product regardless
-      # of the operands' declared signedness (the low 64 bits, and hence a full
-      # 128-bit low result, are identical for signed and unsigned).
+      # `mul` (REX.W F7 /4) multiplies rax by its one operand — b's slot or
+      # promoted register, or rcx when it has neither — into rdx:rax; the high
+      # half lands in rdx, which is stored to the destination. The one-operand
+      # `mul` is the unsigned multiply, so this is the unsigned high product
+      # regardless of the operands' declared signedness (the low 64 bits, and
+      # hence a full 128-bit low result, are identical for signed and unsigned).
       def emit_mulhi(dst, a, b)
-        load_reg(EAX, a)                # rax = a
-        load_reg(ECX, b)                # rcx = b
-        emit(0x48, 0xF7, 0xE1)          # mul rcx  -> rdx:rax = rax * rcx
+        refresh_slot_residency
+        a, b = b, a if prefer_swapped_operands?(a, b)
+        if rm_operand?(b)
+          load_reg(EAX, a)
+          emit_vreg_rm([0xF7], b, reg: 4, rex_w: true) # mul qword r/m64  (/4)
+        else
+          load_binary_operands(a, b, commutative: true)
+          emit(0x48, 0xF7, 0xE1)        # mul rcx  -> rdx:rax = rax * rcx
+        end
         store_reg(EDX, dst)             # dst = high 64 bits
       end
 
-      # A size of 8 does a 64-bit signed division (REX.W cqo + REX.W idiv rcx),
-      # used for pointer differences; otherwise the 32-bit int division.
+      # A size of 8 does a 64-bit signed division (REX.W cqo + REX.W idiv), used
+      # for pointer differences; otherwise the 32-bit int division. The divisor
+      # is the one-operand `idiv`'s r/m, which is b's slot wherever it can be.
       def emit_divmod(dst, a, b, result_reg, size = nil)
-        load_reg(EAX, a)
-        load_reg(ECX, b)
-        if size == 8
-          emit(0x48, 0x99)          # cqo: sign-extend rax into rdx:rax
-          emit(0x48, 0xF7, 0xF9)    # idiv rcx
-        else
-          emit(0x99)                # cdq: sign-extend eax into edx:eax
-          emit(0xF7, 0xF9)          # idiv ecx
-        end
+        divisor_in_register = load_dividend_and_divisor(a, b)
+        emit(0x48) if size == 8    # REX.W turns cdq into cqo
+        emit(0x99)                 # cdq/cqo: sign-extend eax/rax into edx:eax / rdx:rax
+        emit_divisor_operand(divisor_in_register, b, 7, size) # idiv (/7)
         store_reg(result_reg, dst)
       end
 
       # Unsigned division/remainder. Unlike the signed form, the high half of
       # the dividend is zeroed (xor edx, edx, which also clears the upper 32
       # bits of rdx for the 64-bit case) rather than sign-extended, and the
-      # unsigned `div` opcode is used. size 8 divides the full 64-bit rax by
-      # rcx; otherwise the 32-bit division. Quotient in eax, remainder in edx.
+      # unsigned `div` opcode is used. size 8 divides the full 64-bit rax by the
+      # divisor; otherwise the 32-bit division. Quotient in eax, remainder in edx.
       def emit_udivmod(dst, a, b, result_reg, size = nil)
-        load_reg(EAX, a)
-        load_reg(ECX, b)
+        divisor_in_register = load_dividend_and_divisor(a, b)
         emit(0x31, 0xD2)            # xor edx, edx
-        if size == 8
-          emit(0x48, 0xF7, 0xF1)    # div rcx
+        emit_divisor_operand(divisor_in_register, b, 6, size) # div (/6)
+        store_reg(result_reg, dst)
+      end
+
+      # Stages a division's operands: the dividend always into eax, the divisor
+      # into ecx only when it is a transient, which the one-operand `div`/`idiv`
+      # can name no other way. Returns true when the divisor is in ecx. Both
+      # loads happen before the sign-extension step that follows, which writes
+      # edx and must not be undone.
+      #
+      # The dividend and the quotient/remainder pair are fixed at rax and
+      # rdx:rax by the instruction itself, so this is one of the places a
+      # promoted destination changes nothing; a promoted *divisor*, though, is
+      # named in place like any other r/m operand, and rdx cannot collide with
+      # it (PROMOTION_REGISTERS holds none of the scratch registers).
+      def load_dividend_and_divisor(a, b)
+        if rm_operand?(b)
+          load_reg(EAX, a)
+          false
         else
-          emit(0xF7, 0xF1)          # div ecx
+          load_binary_operands(a, b)
+          true
+        end
+      end
+
+      # The r/m operand of the one-operand `div`/`idiv` group (F7 /6 unsigned,
+      # /7 signed): ecx when #load_dividend_and_divisor put it there, and b's
+      # slot or promoted register otherwise.
+      def emit_divisor_operand(in_register, b, extension, size)
+        if in_register
+          emit(0x48) if size == 8
+          emit(0xF7, 0xC0 | (extension << 3) | ECX)
+        else
+          emit_vreg_rm([0xF7], b, reg: extension, rex_w: size == 8)
         end
-        store_reg(result_reg, dst)
       end
 
       # mov r64, [rbp + disp]: slots are always moved 64 bits at a time so a
@@ -1303,17 +1961,146 @@ module Rubycc
       # zero the upper 32 bits of rax, so the slot's high half is already zero.
       # The REX prefix carries W (64-bit operand) plus R for r8/r9 (>= 8), whose
       # low 3 bits go into the ModR/M reg field.
+      #
+      # The load disappears when `reg` already holds this slot's value and
+      # becomes a register-to-register move when another scratch register does
+      # (SlotResidency decides both). Either way `reg` ends up with the same 64
+      # bits the memory form would have fetched — the slot was written 64 bits
+      # at a time and it is those bits being reused — so the value
+      # representation above is untouched, and a narrow value's high half is as
+      # indeterminate (or as zero) as it was before.
       def load_reg(reg, vreg)
-        emit(0x48 | (reg >= 8 ? 0x04 : 0)) # REX.W (+ REX.R for r8/r9)
-        emit(0x8B)
-        emit_modrm_rbp_disp(reg & 7, slot_disp(vreg))
+        refresh_slot_residency
+        promoted = @promoted[vreg]
+        if promoted
+          # There is no slot to read: this value has been in `promoted` since
+          # the prologue and stays there until the function returns, so the
+          # load is a register-to-register move — or nothing at all when the
+          # caller happens to want it where it already is. The move writes
+          # `reg` and touches no memory, which is exactly what the residency
+          # table has to be told (see SlotResidency#note_register_clobbered).
+          emit_reg_move(reg, promoted) unless reg == promoted
+          note_register_clobbered(reg)
+          return
+        end
+        return if slot_resident_in?(reg, vreg)
+
+        source = register_holding_slot(vreg)
+        if source
+          emit_reg_move(reg, source)
+        else
+          emit(0x48 | (reg >= 8 ? 0x04 : 0)) # REX.W (+ REX.R for r8/r9)
+          emit(0x8B)
+          emit_modrm_rbp_disp(reg & 7, slot_disp(vreg))
+        end
+        note_slot_loaded(reg, vreg)
       end
 
       # mov [rbp + disp], r64. See load_reg for the 64-bit and REX rationale.
+      # The store itself is never skipped: the slot is the value's home, and
+      # nothing here knows whether a later branch reaches a reader by a path
+      # that goes nowhere near this register.
       def store_reg(reg, vreg)
+        refresh_slot_residency
+        promoted = @promoted[vreg]
+        if promoted
+          # The value's home is a register, so the store is a move into it. No
+          # slot is written and no scratch register can be describing one (a
+          # promoted vreg is never recorded), so every residency in flight is
+          # still true — it is only the emitted bytes that have to be accounted
+          # for (SlotResidency#note_slots_undisturbed).
+          emit_reg_move(promoted, reg) unless reg == promoted
+          note_slots_undisturbed
+          return
+        end
+        if @transient[vreg]
+          # The value's only reader is the next instruction, which will find it
+          # here; the slot itself is never named again, so nothing is written.
+          note_slot_loaded(reg, vreg)
+          return
+        end
         emit(0x48 | (reg >= 8 ? 0x04 : 0)) # REX.W (+ REX.R for r8/r9)
         emit(0x89)
         emit_modrm_rbp_disp(reg & 7, slot_disp(vreg))
+        note_slot_stored(reg, vreg)
+      end
+
+      # Whether `vreg`'s slot may be named as an instruction's memory operand.
+      # It may not when the value never reached it: a transient is left in the
+      # register its producer computed it in, and a promoted value lives in a
+      # register of its own for the whole function, so in both cases the slot
+      # holds nothing. The test is on the two sets, not on what happens to be
+      # resident, so it does not depend on how far into an instruction it is
+      # asked.
+      #
+      # Only the floating ops ask any more; the integer ones ask #rm_operand?
+      # instead, a promoted value being nameable there as a register even though
+      # its slot is not. The difference does not arise in practice — a value the
+      # vector register file touches is refused promotion outright
+      # (IR::Promotion::VECTOR_OPS), so a floating operand is only ever a slot or
+      # a transient — and the stricter test is the one to ask on a path that has
+      # no register form to fall back to but xmm1.
+      def slot_written?(vreg)
+        !@transient[vreg] && !@promoted[vreg]
+      end
+
+      # mov r64, r64 — the register-to-register form of #load_reg's mov, for a
+      # value that is already in another scratch register or that lives in a
+      # promoted one. REX.W widens it, REX.R extends the source (the ModR/M reg
+      # field) and REX.B the destination (the rm field), so r8/r9 and r12..r15
+      # work on either side. The ModR/M is register-direct (mod = 11), where an
+      # rm of 100 names r12 rather than selecting a SIB byte, so no promoted
+      # register needs a special case the way a memory operand's base would.
+      def emit_reg_move(dst, src)
+        emit(0x48 | (src >= 8 ? 0x04 : 0) | (dst >= 8 ? 0x01 : 0))
+        emit(0x89, 0xC0 | ((src & 7) << 3) | (dst & 7))
+      end
+
+      # Loads a two-operand instruction's operands into eax and ecx.
+      #
+      # Which one is fetched first stops being arbitrary once #load_reg can
+      # reuse a resident value: if `b` is the value sitting in eax, loading `a`
+      # there first would throw it away and force `b` to be re-read from memory.
+      # Fetching ecx first instead keeps it, as a register move. A `commutative`
+      # op can do better still — swapping the two makes the resident operand
+      # eax's, so nothing is moved at all — and is safe to swap precisely
+      # because the opcode's two operands are interchangeable (this is not true
+      # of :sub or of the shifts, whose count must reach cl).
+      def load_binary_operands(a, b, commutative: false)
+        refresh_slot_residency
+        if commutative && prefer_swapped_operands?(a, b)
+          a, b = b, a
+        elsif slot_resident_in?(EAX, b) && !slot_resident_in?(EAX, a)
+          # Fetch ecx first: eax is where b is, and loading a would lose it —
+          # which for a transient b would lose it for good, its slot never
+          # having been written.
+          load_reg(ECX, b)
+        end
+        load_reg(EAX, a)
+        load_reg(ECX, b)
+      end
+
+      # Whether a commutative op should exchange its operands. A promoted
+      # destination outranks everything else: "sum = x + sum" is the same
+      # addition as "sum = sum + x", and only the second can be accumulated in
+      # place (#emit_promoted_binary), which saves a move whatever the operands
+      # cost. With that settled either way, the older question stands alone.
+      def swap_binary_operands?(dst, a, b)
+        return dst == b && dst != a if @promoted[dst]
+
+        prefer_swapped_operands?(a, b)
+      end
+
+      # Whether a commutative op is better off with its operands exchanged.
+      # Two cases want it, and they are one case seen twice: whichever operand
+      # is already in eax should be the one taken from a register, leaving the
+      # other free to be read where it lives. An operand that can be named
+      # neither in a slot nor in a promoted register (a transient) has to be
+      # that one, since no r/m field can reach it.
+      def prefer_swapped_operands?(a, b)
+        return false unless rm_operand?(a)
+
+        !rm_operand?(b) || (slot_resident_in?(EAX, b) && !slot_resident_in?(EAX, a))
       end
 
       def slot_disp(vreg)