rubycc 1.0.0 → 1.1.0

data/lib/rubycc/backend/aarch64.rb

@@ -2,6 +2,10 @@
 
 require_relative "../compile_error"
 require_relative "../ir/ir"
+require_relative "../ir/analysis"
+require_relative "../ir/promotion"
+require_relative "../ir/simplify"
+require_relative "slot_residency"
 
 module Rubycc
   module Backend
@@ -20,11 +24,12 @@ module Rubycc
 
     # AArch64 (ARM64) code generator, the second backend behind the same
     # IR::Function -> Result contract the x86_64 one honors. It keeps the
-    # spill-everything strategy — every virtual register owns an 8-byte stack
-    # slot and each IR instruction loads its operands into scratch registers,
-    # computes, and stores the result back — but the machine underneath is
-    # entirely different: fixed-length 32-bit instructions, a flat register
-    # file, and load/store addressing that shapes the frame layout.
+    # spill-everything strategy for every value but the promoted one — each
+    # virtual register owns an 8-byte stack slot and each IR instruction loads
+    # its operands into scratch registers, computes, and stores the result back
+    # — but the machine underneath is entirely different: fixed-length 32-bit
+    # instructions, a flat register file, and load/store addressing that shapes
+    # the frame layout.
     #
     # Frame layout (frame-base-relative, positive offsets). Unlike x86_64's
     # rbp-negative displacements, every slot is addressed as [base + off] with a
@@ -32,11 +37,11 @@ module Rubycc
     # 12-bit immediate by the access size (reaching 0..32760 for a 64-bit load)
     # while the signed form is only a 9-bit unscaled window (-256..255) that a
     # modest frame overruns at once. From the base upward the frame holds the
-    # outgoing argument area, the saved frame record (x29/x30), the vreg slots
-    # and the stack objects. A slot whose offset still overflows the scaled
-    # immediate is reached by composing its address into a scratch register with
-    # add-immediate(s) — the path is built in from the start rather than bolted
-    # on for large frames.
+    # outgoing argument area, the saved frame record (x29/x30), the vreg slots,
+    # the stack objects and the promoted registers' save slots. A slot whose
+    # offset still overflows the scaled immediate is reached by composing its
+    # address into a scratch register with add-immediate(s) — the path is built
+    # in from the start rather than bolted on for large frames.
     #
     # The frame base is sp itself in an ordinary function, which costs nothing
     # and leaves x29 free. It is x29 in a function containing :alloca, because
@@ -78,7 +83,8 @@ module Rubycc
     # floating arguments arrive in v0..v7, allocated from a counter of their
     # own, and a floating result comes back in v0. The two sequences are
     # independent, exactly as System V's integer and xmm sequences are, which is
-    # why the IR's :gp/:sse4/:sse8 tags carry over unchanged. The generator
+    # why the IR's :gp/:sse4/:sse8 tags carry over unchanged (:sse16, a whole
+    # 16-byte value in one vector register, is this target's alone). The generator
     # classifies against this target's register budget (IR::CallConvention),
     # so a :gp tag really does mean one of the eight and a :mem tag really does
     # mean the stack — no seventh integer argument is spilled here that AAPCS64
@@ -130,6 +136,13 @@ module Rubycc
     # frame addressable (see #emit_alloca). Every IR op the generator can hand
     # this backend is now lowered; there is nothing left it refuses.
     class AArch64
+      # Every slot access goes through #load_reg / #store_reg, which use this to
+      # skip a reload of a value the previous instruction has just left in a
+      # register. See slot_residency.rb for why "nothing has been emitted since"
+      # is a sufficient safety condition, and #emit_instruction's :label case
+      # for the one state change it cannot see.
+      include SlotResidency
+
       # Result of compiling one function, the same shape the x86_64 backend
       # returns: `bytes` the machine code, `symbols` an array of
       # { name:, offset:, size: }, and `relocations` an array of kind-tagged
@@ -201,6 +214,29 @@ module Rubycc
       FA = 16
       FB = 17
 
+      # The registers whole-function promotion hands out, in the order it hands
+      # them out (see IR::Promotion and #promotion_assignment). All ten are
+      # callee-saved under AAPCS64 (5.1.1: x19..x28 are preserved across a
+      # call), which is what lets a promoted value stay put across one, and all
+      # ten are clear of everything else this backend names: the scratch
+      # registers x9..x15, the argument/result registers x0..x8, the frame
+      # pointer x29 and the link register x30.
+      #
+      # The three callee-saved-looking registers left out are left out on
+      # purpose. x18 is the platform register, reserved by AAPCS64 5.1.1 for
+      # whatever the running platform ABI wants it for, so no compiler may use
+      # it as a general one. x16 and x17 (IP0/IP1) are corruptible by any
+      # branch: the linker is free to insert a veneer at a `bl` site, and a
+      # veneer clobbers them.
+      #
+      # Being disjoint from every register the residency table is keyed by is
+      # what lets an instruction that writes a promoted register leave that
+      # table standing (see SlotResidency#note_register_clobbered), and #load_reg
+      # keeps the disjointness true from the other side by never recording a
+      # residency against one.
+      PROMOTION_REGISTERS = (19..28).to_a.freeze
+      PROMOTION_REGISTER_SET = PROMOTION_REGISTERS.to_set.freeze
+
       # Special register numbers. In the load/store, add/sub-immediate and
       # stp/ldp encodings a register field of 31 denotes the stack pointer; in
       # the data-processing (arithmetic/logical shifted-register) encodings the
@@ -255,8 +291,28 @@ module Rubycc
         flt: 4, fle: 9, fgt: 12, fge: 10
       }.freeze
 
-      def compile(ir_func)
+      # `analysis` is the census of `ir_func`'s instruction list (IR::Analysis).
+      # The compiler passes the one IR::Simplify already took on its way
+      # through; a caller that hands over a function directly gets it taken
+      # here.
+      def compile(ir_func, analysis = IR::Analysis.of(ir_func))
         @code = +"".b
+        reset_slot_residency
+        # Slots this function never has to write: an expression temporary whose
+        # one reader is the instruction right behind its producer stays in the
+        # register it was computed in (see IR::Simplify#transient_flags and
+        # #store_reg). The answer is a property of the instruction list, so it
+        # is taken once here rather than rediscovered per instruction, and it
+        # arrives as an array indexed by vreg number — the numbers are dense and
+        # small, and this is asked on every store the backend emits.
+        @transient = analysis.transient
+        # Slots this function does not have at all: a promoted value lives in
+        # one callee-saved register from the prologue to the epilogue, so its
+        # slot is never read, written or named (see #promotion_assignment,
+        # #load_reg and #store_reg). Like @transient this is a property of the
+        # instruction list, decided once here, and indexed by vreg number for
+        # the same reason.
+        @promoted = promotion_assignment(ir_func, analysis)
         # @labels maps a label id to its resolved byte offset; @fixups collects
         # [patch_offset, label_id, kind] for each forward/backward branch whose
         # immediate is written once every label offset is known.
@@ -282,13 +338,32 @@ module Rubycc
 
       private
 
+      # Binds the best promotion candidates to PROMOTION_REGISTERS in order,
+      # returning the vreg -> register table the whole backend consults (an
+      # array; a vreg that owns no register indexes nil). Taking the first few
+      # of an ordered list is the entire allocation: one register each, held for
+      # the function's length, so there is no interference to resolve and
+      # nothing to undo when the registers run out — the candidates past the
+      # tenth simply keep their slots.
+      #
+      # @promoted_registers is the same binding in assignment order, which is
+      # the order the save slots are laid out in (#layout_frame).
+      def promotion_assignment(ir_func, analysis)
+        vregs = IR::Promotion.candidates(ir_func, analysis).first(PROMOTION_REGISTERS.size)
+        @promoted_registers = PROMOTION_REGISTERS.first(vregs.size)
+        promoted = Array.new(ir_func.vreg_count)
+        vregs.each_with_index { |vreg, index| promoted[vreg] = PROMOTION_REGISTERS[index] }
+        promoted
+      end
+
       # Computes the frame's size and every object's base offset. From sp
       # upward: the outgoing argument area (empty unless some call in this
       # function passes an argument on the stack), the 16-byte saved record,
       # the vreg slots (8 bytes each, the region rounded to 16 so the objects
       # stay 16-aligned), then each stack object at a 16-byte-aligned size above
-      # the previous. All offsets are non-negative displacements from the frame
-      # base, which this method also decides (see #frame_base_register).
+      # the previous, then one 8-byte save slot per promoted register. All
+      # offsets are non-negative displacements from the frame base, which this
+      # method also decides (see #frame_base_register).
       def layout_frame(vreg_count, stack_objects, insts, variadic)
         # Whether this function allocates dynamically decides two things at
         # once: which register names the fixed frame (see #frame_base_register)
@@ -309,6 +384,28 @@ module Rubycc
           @object_offsets << running
           running += align16(object_size)
         end
+        # One save slot per promoted register, above every object. The
+        # registers are callee-saved, so a function that takes one over has to
+        # give it back exactly as it found it. This end of the frame is the one
+        # place a new region can go without moving anything: the outgoing
+        # argument area has to stay at the bottom (AAPCS64 has the callee read
+        # its stack arguments from the sp of the `bl`), the saved record and the
+        # vreg slots are what every offset in the body is measured against, and
+        # the objects sit between. Putting the saves last therefore leaves every
+        # other offset exactly what it would have been without promotion. Their
+        # number is odd as often as not, but nothing above them cares: the one
+        # region that has to end exactly at the frame's top is the variadic
+        # register-save area, and a variadic function has no promoted register
+        # at all (IR::Promotion refuses one), so the two never meet. The single
+        # align16 below is what fixes the frame size, so sp stays 16-aligned
+        # whatever the count. They are laid out in assignment order,
+        # a function of the instruction list, so the frame stays deterministic
+        # (N4).
+        @promoted_saves = @promoted_registers.map do |reg|
+          offset = running
+          running += 8
+          [reg, offset]
+        end
         # A variadic function reserves the argument register-save area at the
         # very top of the frame — just below the caller's sp, exactly where the
         # AArch64 C library expects __gr_top/__vr_top to point. The vector area
@@ -333,12 +430,18 @@ module Rubycc
       # so the saved record above it — and sp itself at every call — stays
       # 16-aligned. Sizing it for the widest call lets every call share the one
       # area, since only one call is in flight at a time.
+      #
+      # A :pad_stack slot occupies an eightbyte of the area exactly as a :mem
+      # one does — it is the gap that 16-aligns the argument behind it — so it
+      # is counted here on the same footing. #place_arguments counts the two
+      # together as well, and the two counts have to agree or a call would write
+      # past the area the frame reserved for it.
       def outgoing_argument_bytes(insts)
         widest = 0
         insts.each do |inst|
           next unless inst.op == :call || inst.op == :call_indirect
 
-          count = inst.b.count { |_vreg, kind| kind == :mem }
+          count = inst.b.count { |_vreg, kind| kind == :mem || kind == :pad_stack }
           widest = count if count > widest
         end
         align16(widest * 8)
@@ -377,10 +480,32 @@ module Rubycc
         adjust_sp(@frame_size, sub: true)
         emit_save_record(store: true, base: SP)
         emit_add_imm(FP, SP, 0, shift12: false) if @uses_alloca # mov x29, sp
+        # Before anything writes a promoted register — which the parameter
+        # spilling right below is the first thing to do, a promoted parameter
+        # being moved straight from the register it arrived in — and after the
+        # frame base is established, since the saves are addressed against it.
+        emit_save_promoted_registers
         spill_parameters(param_kinds)
         save_argument_registers if variadic
       end
 
+      # Saves the promoted registers into their frame slots, one "str Xn,
+      # [base + off]" apiece. A store into the frame rather than a push (or an
+      # stp pair) keeps sp's 16-byte alignment a property of the single frame
+      # adjustment in the prologue instead of the parity of the register count.
+      def emit_save_promoted_registers
+        @promoted_saves.each { |reg, offset| store_frame_at(reg, offset) }
+      end
+
+      # Puts the caller's values back, the exact inverse of
+      # #emit_save_promoted_registers. #emit_epilogue carries it, and an
+      # epilogue is emitted at every :ret rather than once per function, so a
+      # function with several returns restores on each of them and no path
+      # leaves with a callee-saved register holding this function's value.
+      def emit_restore_promoted_registers
+        @promoted_saves.each { |reg, offset| load_frame_at(reg, offset) }
+      end
+
       # Spills every argument register into the variadic save area so a later
       # :va_start can hand __builtin_va_arg a pointer to each. The eight integer
       # registers go into the 8-byte slots at the top of the frame and the eight
@@ -447,8 +572,16 @@ module Rubycc
       # the end of the scope that allocated it. The record is then reloaded and
       # the fixed frame released exactly as in an ordinary function (sp and x29
       # are equal by then, so the reload needs no special base).
+      #
+      # The promoted registers are given back here, after the return value has
+      # been loaded (it may itself come out of one) and — in an alloca function
+      # — after sp has been brought back to the fixed frame, which is the
+      # moment the save slots become addressable again through sp. They are in
+      # fact reached through x29 there, so the order is not what makes this
+      # correct; it is what keeps the two cases reading the same way.
       def emit_epilogue
         emit_add_imm(SP, FP, 0, shift12: false) if @uses_alloca # mov sp, x29
+        emit_restore_promoted_registers
         emit_save_record(store: false)
         adjust_sp(@frame_size, sub: false)
         emit_word(0xD65F03C0) # ret (branch to x30)
@@ -470,12 +603,14 @@ module Rubycc
         end
       end
 
-      # Brings each incoming argument into its parameter slot so it reads back
-      # like any other vreg. Integer/pointer parameters come out of x0..x7 and
-      # floating ones out of v0..v7, each sequence advancing its own counter; a
-      # :mem parameter is copied down from the caller's stack argument area
-      # (through the A scratch, which is not an argument register, so the copies
-      # never disturb an argument still to be spilled) as a whole eightbyte,
+      # Brings each incoming argument into its home — its parameter slot, or
+      # the callee-saved register it was promoted into, which #store_reg
+      # decides — so it reads back like any other vreg. Integer/pointer
+      # parameters come out of x0..x7 and floating ones out of v0..v7, each
+      # sequence advancing its own counter; a :mem parameter is copied down
+      # from the caller's stack argument area (through the A scratch, which is
+      # not an argument register, so the copies never disturb an argument still
+      # to be spilled, or through its promoted register) as a whole eightbyte,
       # which is right for a narrow or floating value too since the slot
       # discipline only promises its low bytes. A kind that would overrun its
       # register file is a generator contract violation and raises.
@@ -491,8 +626,14 @@ module Rubycc
             store_reg(ARG_REGISTERS[next_gp], i)
             next_gp += 1
           when :mem
-            load_frame_at(A, incoming_stack_offset(next_stack))
-            store_reg(A, i)
+            # Straight into the register a promoted parameter lives in: the
+            # copy down from the caller's area is a load like any other, and
+            # its destination is a free field, so nothing is gained by routing
+            # it through A first. #store_reg then emits nothing at all, the
+            # value being already home.
+            target = @promoted[i] || A
+            load_frame_at(target, incoming_stack_offset(next_stack))
+            store_reg(target, i)
             next_stack += 1
           when :pad
             # An even-pair alignment pad consumes one integer register but binds
@@ -520,14 +661,23 @@ module Rubycc
       end
 
       def emit_instruction(inst)
+        # Once, at the top, rather than in each lowering that consults the
+        # table. An instruction whose operands are all promoted loads nothing,
+        # so it can reach #store_result — which revalidates the table — without
+        # any of the loads that would otherwise have discarded a stale one
+        # first. Refreshing here means the table is either true or empty by the
+        # time any lowering touches it, whatever the instruction turns out to
+        # be.
+        refresh_slot_residency
         case inst.op
         when :const then emit_const(inst.dst, inst.a, inst.size)
         when :copy then emit_copy(inst.dst, inst.a)
-        when :add then emit_arith(inst, ADD_SHIFTED)
+        when :add then emit_arith(inst, ADD_SHIFTED, commutative: true)
+        when :scaled_add then emit_scaled_add(inst.dst, inst.a, inst.b, inst.size)
         when :sub then emit_arith(inst, SUB_SHIFTED)
-        when :and then emit_arith(inst, AND_SHIFTED)
-        when :or then emit_arith(inst, ORR_SHIFTED)
-        when :xor then emit_arith(inst, EOR_SHIFTED)
+        when :and then emit_arith(inst, AND_SHIFTED, commutative: true)
+        when :or then emit_arith(inst, ORR_SHIFTED, commutative: true)
+        when :xor then emit_arith(inst, EOR_SHIFTED, commutative: true)
         when :mul then emit_mul(inst.dst, inst.a, inst.b, inst.size)
         when :div then emit_divmod(inst.dst, inst.a, inst.b, inst.size, signed: true, remainder: false)
         when :mod then emit_divmod(inst.dst, inst.a, inst.b, inst.size, signed: true, remainder: true)
@@ -538,10 +688,17 @@ module Rubycc
         when :shr then emit_shift(inst.dst, inst.a, inst.b, inst.size, LSRV)
         when :neg then emit_neg(inst.dst, inst.a, inst.size)
         when :eq, :ne, :lt, :le, :gt, :ge, :ult, :ule, :ugt, :uge
-          emit_comparison(inst.dst, inst.a, inst.b, CONDITIONS.fetch(inst.op), inst.size)
+          emit_comparison(inst.dst, inst.a, inst.b, CONDITIONS.fetch(inst.op), inst.size,
+                          commutative: inst.op == :eq || inst.op == :ne)
         when :sext then emit_sext(inst.dst, inst.a, inst.size)
         when :zext then emit_zext(inst.dst, inst.a, inst.size)
-        when :label then @labels[inst.a] = @code.bytesize
+        when :label
+          # The one place a register's meaning changes without an instruction
+          # being emitted: control may arrive here from a branch whose registers
+          # hold something else entirely, so nothing may be assumed resident
+          # past a label (see SlotResidency).
+          forget_slot_residency
+          @labels[inst.a] = @code.bytesize
         when :jump then emit_b(inst.a)
         when :jump_if_zero then emit_jump_if_zero(inst.a, inst.b)
         when :call then emit_call(inst.dst, inst.a, inst.b, inst.size)
@@ -563,6 +720,7 @@ module Rubycc
         when :call_indirect then emit_call_indirect(inst.dst, inst.a, inst.b, inst.size)
         when :mulhi then emit_mulhi(inst.dst, inst.a, inst.b)
         when :bit_scan then emit_bit_scan(inst.dst, inst.a, inst.b, inst.size)
+        when :popcount then emit_popcount(inst.dst, inst.a, inst.size)
         when :string_addr then emit_symbol_address(inst.dst, kind: :string, string_id: inst.a)
         when :global_addr then emit_symbol_address(inst.dst, kind: :global, symbol: inst.a)
         when :func_addr then emit_symbol_address(inst.dst, kind: :func, symbol: inst.a)
@@ -585,34 +743,52 @@ module Rubycc
       # 32-bit value whose W-register move zeroes the slot's high half, matching
       # the x86_64 backend's "mov eax, imm32" behavior.
       def emit_const(dst, value, size)
-        materialize(A, value, size == 8 ? 64 : 32)
-        store_reg(A, dst)
+        target = result_register(dst)
+        materialize(target, value, size == 8 ? 64 : 32)
+        store_result(target, dst, only_wrote: target) # movz/movk write target alone
       end
 
-      # :copy — a 64-bit slot-to-slot transfer.
+      # :copy — a 64-bit transfer between the two values' homes, whichever
+      # those are: slot to slot, slot to register, register to slot, or one
+      # `mov` between two promoted registers.
       def emit_copy(dst, src)
-        load_reg(A, src)
-        store_reg(A, dst)
-      end
-
-      # An arithmetic/logical binary op. The operands are loaded into A/B and
-      # combined with a shifted-register instruction whose width follows the IR
-      # size (64-bit for size 8, otherwise a 32-bit W-register op whose upper
-      # half is zeroed for free).
-      def emit_arith(inst, base_table)
-        load_reg(A, inst.a)
-        load_reg(B, inst.b)
-        emit_word(base_table[width(inst.size)] | (B << 16) | (A << 5) | A)
-        store_reg(A, inst.dst)
+        target = result_register(dst)
+        load_reg(target, src)
+        store_result(target, dst, only_wrote: target) # the load wrote target, or nothing
+      end
+
+      # An arithmetic/logical binary op, combined with a shifted-register
+      # instruction whose width follows the IR size (64-bit for size 8,
+      # otherwise a 32-bit W-register op whose upper half is zeroed for free).
+      # All three of Rd, Rn and Rm are free fields, so each end is named
+      # wherever it lives: a promoted destination and two promoted operands
+      # make the whole instruction one word with no traffic around it.
+      def emit_arith(inst, base_table, commutative: false)
+        rn, rm = binary_operand_registers(inst.a, inst.b, commutative: commutative)
+        rd = result_register(inst.dst)
+        emit_word(base_table[width(inst.size)] | (rm << 16) | (rn << 5) | rd)
+        store_result(rd, inst.dst, only_wrote: rd) # one word, Rd its only written field
+      end
+
+      # :scaled_add — dst <- base + index * element_size, the address a
+      # subscript forms. The shifted-register add carries the scale in its imm6
+      # field as a left shift of the second operand, so the element-size
+      # multiply and the add to the base are one instruction. It is always the
+      # 64-bit (X) form: the value being computed is a pointer.
+      def emit_scaled_add(dst, base_vreg, index_vreg, element_size)
+        rn, rm = binary_operand_registers(base_vreg, index_vreg)
+        rd = result_register(dst)
+        shift = SHIFTED_SCALES.fetch(element_size)
+        emit_word(ADD_SHIFTED[64] | (rm << 16) | (shift << 10) | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # :mul — Rd = Rn * Rm, encoded as `madd Rd, Rn, Rm, xzr`.
       def emit_mul(dst, a, b, size)
-        load_reg(A, a)
-        load_reg(B, b)
-        base = size == 8 ? 0x9B007C00 : 0x1B007C00 # madd with Ra = xzr
-        emit_word(base | (B << 16) | (A << 5) | A)
-        store_reg(A, dst)
+        rn, rm = binary_operand_registers(a, b, commutative: true)
+        rd = result_register(dst)
+        emit_word(MADD_ZERO[width(size)] | (rm << 16) | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # :mulhi — the unsigned high 64 bits of a 64x64 product, encoded as
@@ -623,10 +799,10 @@ module Rubycc
       # of-a-32x32-product need for it here), so this always runs on the X
       # registers regardless of the IR size.
       def emit_mulhi(dst, a, b)
-        load_reg(A, a)
-        load_reg(B, b)
-        emit_word(UMULH | (B << 16) | (A << 5) | A)
-        store_reg(A, dst)
+        rn, rm = binary_operand_registers(a, b, commutative: true)
+        rd = result_register(dst)
+        emit_word(UMULH | (rm << 16) | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # :bit_scan — count the zero bits of a's value (__builtin_ctz/clz and
@@ -652,12 +828,69 @@ module Rubycc
       # promises nothing there and the x86-64 backend cannot match it anyway.
       def emit_bit_scan(dst, src_vreg, direction, size)
         w = width(size)
-        load_reg(A, src_vreg)
-        emit_word(RBIT[w] | (A << 5) | A) if direction == :forward
-        emit_word(CLZ[w] | (A << 5) | A)
+        rn = operand_register(src_vreg, A)
+        rd = result_register(dst)
+        if direction == :forward
+          emit_word(RBIT[w] | (rn << 5) | rd)
+          rn = rd # the reversal is what CLZ counts, wherever it landed
+        end
+        emit_word(CLZ[w] | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # RBIT and CLZ both write rd and nothing else
+      end
+
+      # :popcount — how many bits of a's value are set (__builtin_popcount and
+      # its "l"/"ll" forms). AArch64's population count, CNT, is an AdvSIMD
+      # instruction: it counts within each byte of a *vector* register, so using
+      # it would mean moving the operand across register files and then summing
+      # the byte counts anyway (ADDV). The general-register expansion below
+      # avoids the crossing entirely, and is the same divide-and-conquer sum
+      # (SWAR) the x86-64 backend emits — where the derivation of the four
+      # stages and of their masks is written out (backend/x86_64.rb
+      # #emit_popcount), that being the one place worth stating it.
+      #
+      # Only the final stage differs in what it can name: the multiplier and the
+      # masks are all bitmask immediates here (repeating bit patterns, which is
+      # exactly what "logical (immediate)" encodes), so no constant has to be
+      # built with movz/movk — the multiply's operand does need a register, but
+      # one `mov` of an immediate fills it.
+      #
+      # A runs the value through the stages and B holds the partner each stage
+      # shifts out of it. A is loaded rather than read where it lies because
+      # every stage overwrites it, so a promoted operand would be destroyed.
+      def emit_popcount(dst, src_vreg, size)
+        w = width(size)
+        load_reg(A, src_vreg)                             # A = value to count
+        emit_popcount_lsr(B, A, 1, w)                     # B = x >> 1
+        emit_popcount_and(B, B, POPCOUNT_PAIR_IMMS, w)    # B &= 0x5555...
+        emit_word(SUB_SHIFTED[w] | (B << 16) | (A << 5) | A) # A = pair counts
+        emit_popcount_and(B, A, POPCOUNT_NIBBLE_IMMS, w)  # B = x & 0x3333...
+        emit_popcount_lsr(A, A, 2, w)                     # A = x >> 2
+        emit_popcount_and(A, A, POPCOUNT_NIBBLE_IMMS, w)  # A &= 0x3333...
+        emit_word(ADD_SHIFTED[w] | (B << 16) | (A << 5) | A) # A = nibble counts
+        emit_popcount_lsr(B, A, 4, w)                     # B = x >> 4
+        emit_word(ADD_SHIFTED[w] | (B << 16) | (A << 5) | A) # A = x + (x >> 4)
+        emit_popcount_and(A, A, POPCOUNT_BYTE_IMMS, w)    # A = one count per byte
+        emit_word(MOV_IMM[w] | (POPCOUNT_ONES_IMMS << 10) | (XZR << 5) | B) # B = 0x0101...
+        emit_word(MADD_ZERO[w] | (B << 16) | (A << 5) | A) # A = the bytes, summed
+        emit_popcount_lsr(A, A, size * 8 - 8, w)          # A = the top byte
         store_reg(A, dst)
       end
 
+      # `lsr Rd, Rn, #shift` at the count's width: the shift-by-immediate the
+      # architecture has no opcode of its own for, being the UBFM that keeps the
+      # field from `shift` up to the register's top bit and right-aligns it.
+      def emit_popcount_lsr(rd, rn, shift, w)
+        emit_word(LSR_IMM[w] | (shift << 16) | ((w - 1) << 10) | (rn << 5) | rd)
+      end
+
+      # `and Rd, Rn, #mask` at the count's width, the mask named by the imms
+      # field that encodes its repeating pattern (see POPCOUNT_PAIR_IMMS). Every
+      # mask this expansion needs starts its run of ones at bit 0, so immr — the
+      # rotation — is zero throughout and is not a parameter.
+      def emit_popcount_and(rd, rn, imms, w)
+        emit_word(AND_IMM[w] | (imms << 10) | (rn << 5) | rd)
+      end
+
       # :alloca — dynamic stack allocation (__builtin_alloca). The requested
       # byte count is rounded up to a multiple of 16 (add 15, then clear the low
       # four bits), sp is lowered by that much, and the resulting sp — the
@@ -686,92 +919,126 @@ module Rubycc
         emit_add_imm(A, A, 15, shift12: false)        # add A, A, #15
         emit_word(AND_NOT15 | (A << 5) | A)           # and A, A, #-16
         emit_word(SUB_EXTENDED | (A << 16) | (SP << 5) | SP) # sub sp, sp, A
-        emit_add_imm(A, SP, 0, shift12: false)        # mov A, sp (the block base)
-        store_reg(A, dst)
-      end
-
-      # :div/:mod/:udiv/:umod. The quotient is `sdiv`/`udiv` of A by B; a
-      # remainder is then A - quotient*B via `msub`, since AArch64 has no direct
-      # remainder instruction. The signed/unsigned split mirrors the IR's own.
+        # The rounded count in A is dead once sp has moved, so the block's base
+        # address goes straight to wherever the destination lives.
+        target = result_register(dst)
+        emit_add_imm(target, SP, 0, shift12: false)    # mov target, sp
+        # No `only_wrote:`: the rounding above overwrote A, which the load a
+        # line earlier left holding the *requested* count and which any
+        # residency in flight may name. Claiming otherwise would leave a reader
+        # of the size value believing A still held it (see #store_result).
+        store_result(target, dst)
+      end
+
+      # :div/:mod/:udiv/:umod. The quotient is `sdiv`/`udiv` of the dividend by
+      # the divisor; a remainder is then dividend - quotient*divisor via
+      # `msub`, since AArch64 has no direct remainder instruction. The
+      # signed/unsigned split mirrors the IR's own.
+      #
+      # A remainder needs the quotient as well as the result, so that one keeps
+      # the C scratch for it; C is never a promotion register, so the divisor
+      # and dividend it is read alongside can be named in place regardless.
       def emit_divmod(dst, a, b, size, signed:, remainder:)
-        load_reg(A, a)
-        load_reg(B, b)
+        rn, rm = binary_operand_registers(a, b)
         w = width(size)
         div_base = signed ? (w == 64 ? 0x9AC00C00 : 0x1AC00C00) : (w == 64 ? 0x9AC00800 : 0x1AC00800)
-        emit_word(div_base | (B << 16) | (A << 5) | C) # sdiv/udiv C, A, B
         if remainder
+          rd = result_register(dst)
+          emit_word(div_base | (rm << 16) | (rn << 5) | C)            # sdiv/udiv C, a, b
           msub_base = w == 64 ? 0x9B008000 : 0x1B008000
-          emit_word(msub_base | (B << 16) | (A << 10) | (C << 5) | A) # msub A, C, B, A
-          store_reg(A, dst)
+          emit_word(msub_base | (rm << 16) | (rn << 10) | (C << 5) | rd) # msub rd, C, b, a
+          # No `only_wrote:`: the quotient went to C, which is a scratch a
+          # residency may name — the quotient path right below computes into it
+          # whenever its destination lives in a slot (see #store_result).
+          store_result(rd, dst)
         else
-          store_reg(C, dst)
+          rd = result_register(dst, C)
+          emit_word(div_base | (rm << 16) | (rn << 5) | rd)           # sdiv/udiv rd, a, b
+          store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
         end
       end
 
-      # :shl/:sar/:shr — a variable shift by B's low bits (masked to 5 bits for
-      # a 32-bit operand, 6 for a 64-bit one by the hardware, matching C).
+      # :shl/:sar/:shr — a variable shift by the second operand's low bits
+      # (masked to 5 bits for a 32-bit operand, 6 for a 64-bit one by the
+      # hardware, matching C). The shift amount is an ordinary Rm here, not the
+      # one fixed register x86 makes of it, so a promoted count needs no move.
       def emit_shift(dst, a, b, size, base_table)
-        load_reg(A, a)
-        load_reg(B, b)
-        emit_word(base_table[width(size)] | (B << 16) | (A << 5) | A)
-        store_reg(A, dst)
+        rn, rm = binary_operand_registers(a, b)
+        rd = result_register(dst)
+        emit_word(base_table[width(size)] | (rm << 16) | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # :neg — Rd = -a, encoded as `sub Rd, xzr, a`.
       def emit_neg(dst, src, size)
-        load_reg(A, src)
-        emit_word(SUB_SHIFTED[width(size)] | (A << 16) | (XZR << 5) | A)
-        store_reg(A, dst)
+        rn = operand_register(src, A)
+        rd = result_register(dst)
+        emit_word(SUB_SHIFTED[width(size)] | (rn << 16) | (XZR << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # A comparison materialized into the destination as an int 0/1. `cmp`
       # sets the flags; `cset` then writes 1 when the condition holds. A size-8
       # comparison uses the 64-bit X view (full pointer values), otherwise the
       # 32-bit W view.
-      def emit_comparison(dst, a, b, condition, size)
-        load_reg(A, a)
-        load_reg(B, b)
-        emit_word(SUBS_SHIFTED[width(size)] | (B << 16) | (A << 5) | XZR) # cmp A, B (subs xzr, A, B)
-        emit_cset(A, condition)
-        store_reg(A, dst)
+      #
+      # Both ends are read-only fields and `cset`'s destination is free, so a
+      # comparison between two promoted values that lands in a third is three
+      # registers named in place and no traffic at all.
+      def emit_comparison(dst, a, b, condition, size, commutative: false)
+        rn, rm = binary_operand_registers(a, b, commutative: commutative)
+        rd = result_register(dst)
+        emit_word(SUBS_SHIFTED[width(size)] | (rm << 16) | (rn << 5) | XZR) # cmp rn, rm
+        emit_cset(rd, condition)
+        # The `cmp` writes the flags and register 31, which is xzr here and no
+        # register at all; the `cset` writes rd. Nothing the table names.
+        store_result(rd, dst, only_wrote: rd)
       end
 
       # :sext — sign-extend a's low `size` bytes to the full 64-bit register,
       # so a subsequent 64-bit use (pointer-offset scaling) sees a correct,
       # possibly negative, value.
       def emit_sext(dst, src, size)
-        load_reg(A, src)
+        rn = operand_register(src, A)
+        rd = result_register(dst)
         word =
           case size
           when 1 then 0x93401C00 # sxtb x, w
           when 2 then 0x93403C00 # sxth x, w
           else 0x93407C00        # sxtw x, w  (size 4)
           end
-        emit_word(word | (A << 5) | A)
-        store_reg(A, dst)
+        emit_word(word | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # :zext — zero-extend a's low `size` bytes. The 1/2-byte forms are 32-bit
       # uxtb/uxth (which zero the upper 32 bits too); size 4 is a 64-bit ubfx of
       # the low 32 bits, matching x86's "mov eax, eax".
       def emit_zext(dst, src, size)
-        load_reg(A, src)
+        rn = operand_register(src, A)
+        rd = result_register(dst)
         word =
           case size
           when 1 then 0x53001C00 # uxtb w, w
           when 2 then 0x53003C00 # uxth w, w
           else 0xD3407C00        # ubfx x, x, #0, #32  (size 4)
           end
-        emit_word(word | (A << 5) | A)
-        store_reg(A, dst)
+        emit_word(word | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # one word, Rd its only written field
       end
 
       # :load / :uload — read `size` bytes through the pointer in a's slot. A
       # signed load sign-extends a byte/halfword (ldrsb/ldrsh), the unsigned
       # form zero-extends (ldrb/ldrh); a 4-byte load is a plain W load (upper
       # half zeroed) and an 8-byte load a full X load, both sign-agnostic.
+      #
+      # The base register Rn is an ordinary five-bit field, so a promoted
+      # pointer addresses memory where it stands — the extension x86_64's
+      # ModR/M and SIB special cases made worth deferring there and which costs
+      # nothing here.
       def emit_load(dst, ptr, size, signed:)
-        load_reg(A, ptr) # A = pointer value
+        rn = operand_register(ptr, A)
+        rd = result_register(dst)
         word =
           case size
           when 8 then 0xF9400000                  # ldr  x, [x]
@@ -779,16 +1046,16 @@ module Rubycc
           when 1 then signed ? 0x39C00000 : 0x39400000 # ldrsb/ldrb w, [x]
           else 0xB9400000                         # ldr  w, [x]  (size 4)
           end
-        emit_word(word | (A << 5) | A)
-        store_reg(A, dst)
+        emit_word(word | (rn << 5) | rd)
+        store_result(rd, dst, only_wrote: rd) # the load writes Rt; its base is read only
       end
 
       # :store — write value b's low `size` bytes through the pointer in a. The
       # narrower stores (strb/strh/str-w) are exactly the truncation a narrow
-      # lvalue needs.
+      # lvalue needs. Base and source register are both free fields, so a
+      # promoted pointer and a promoted value make this one instruction.
       def emit_store(ptr, value, size)
-        load_reg(A, ptr)   # A = destination address
-        load_reg(B, value) # B = value
+        rn, rt = binary_operand_registers(ptr, value)
         word =
           case size
           when 8 then 0xF9000000 # str  x, [x]
@@ -796,17 +1063,21 @@ module Rubycc
           when 1 then 0x39000000 # strb w, [x]
           else 0xB9000000        # str  w, [x]  (size 4)
           end
-        emit_word(word | (A << 5) | B)
+        emit_word(word | (rn << 5) | rt)
       end
 
       # :jump_if_zero — branch when a's low 32 bits are zero. Testing the W view
       # (cbz w) mirrors the x86_64 backend's 32-bit "test eax, eax": the
       # condition is an int 0/1 or a truthiness test the generator has already
       # reduced.
+      #
+      # CBZ names the register it tests in a free field, so a promoted
+      # condition is branched on where it lives and the loop's test costs one
+      # instruction.
       def emit_jump_if_zero(cond, label_id)
-        load_reg(A, cond)
+        rt = operand_register(cond, A)
         @fixups << [@code.bytesize, label_id, :cbz]
-        emit_word(0x34000000 | A) # cbz w{A}, <patched>
+        emit_word(0x34000000 | rt) # cbz w{rt}, <patched>
       end
 
       # :call — a direct call. Arguments are placed in x0..x7 / v0..v7, then `bl`
@@ -944,6 +1215,17 @@ module Rubycc
 
             load_fp(FP_ARG_REGISTERS[next_fp], vreg, kind == :sse8 ? 8 : 4)
             next_fp += 1
+          when :sse16
+            raise "call argument #{kind} overruns the vector registers" if next_fp >= FP_ARG_REGISTERS.size
+
+            # A quad-precision argument (AAPCS64's `long double`) fills a whole
+            # vector register, which no eightbyte slot could have held: its
+            # vreg carries the *address* of the 16-byte value instead, and the
+            # register is loaded from there. A is free here — the stack pass
+            # above has finished with it and no argument register is A.
+            load_reg(A, vreg)
+            emit_word(LDR_Q | (A << 5) | FP_ARG_REGISTERS[next_fp]) # ldr q, [A]
+            next_fp += 1
           when :indirect_result
             # The result buffer's address goes in x8, outside both argument
             # sequences, so it never displaces a real argument.
@@ -1024,8 +1306,10 @@ module Rubycc
       # flight. None is an argument register, so a copy made while a call's
       # arguments are being prepared never disturbs one already placed.
       def emit_memcpy(dest_vreg, src_vreg, byte_count)
-        load_reg(A, dest_vreg) # A = destination address
-        load_reg(B, src_vreg)  # B = source address
+        # Through #load_binary_operands rather than two bare loads: the source
+        # address may be a transient waiting in A, which filling A with the
+        # destination would destroy for good (its slot was never written).
+        load_binary_operands(dest_vreg, src_vreg) # A = destination, B = source
         chunks = byte_count / 8
         if chunks > MEMCPY_UNROLL_LIMIT
           emit_memcpy_loop(chunks)
@@ -1231,13 +1515,216 @@ module Rubycc
       # is never truncated; a 32-bit value's high half was zeroed when it was
       # produced. Reaches the slot through the scaled immediate when it fits,
       # otherwise through a composed address in the ADDR scratch.
+      #
+      # The load disappears when `reg` already holds this slot's value and
+      # becomes a register move when another scratch register does (see
+      # SlotResidency). ADDR is never tracked, so the address composition the
+      # distant-slot path performs in it cannot leave a stale claim behind: it
+      # is a scratch this method writes and reads within one instruction and
+      # #load_reg is never asked to fill it.
       def load_reg(reg, vreg)
-        load_frame_at(reg, slot_offset(vreg))
+        refresh_slot_residency
+        promoted = @promoted[vreg]
+        if promoted
+          # There is no slot to read: this value has been in `promoted` since
+          # the prologue and stays there until the function returns, so the
+          # load is a register move — or nothing at all when the caller wants
+          # it where it already is. Either way only `reg` is written, which is
+          # what the residency table has to be told.
+          emit_move_register(reg, promoted) unless reg == promoted
+          note_register_clobbered(reg)
+          return
+        end
+        return if slot_resident_in?(reg, vreg)
+
+        source = register_holding_slot(vreg)
+        if source
+          emit_move_register(reg, source)
+        else
+          load_frame_at(reg, slot_offset(vreg))
+        end
+        if PROMOTION_REGISTER_SET.include?(reg)
+          # Filling a promoted register from some *other* value's slot happens
+          # where that register is a result destination and the value is on its
+          # way in (#emit_copy, a stack parameter). Recording a residency
+          # against it would break the invariant every note_register_clobbered
+          # in this backend rests on — that no entry in the table is keyed by a
+          # promotion register — and would be short-lived anyway, the result
+          # being written over it in the next instruction.
+          note_register_clobbered(reg)
+        else
+          note_slot_loaded(reg, vreg)
+        end
+      end
+
+      # mov Xd, Xn, which the architecture spells as an ORR with the zero
+      # register.
+      def emit_move_register(dst, src)
+        emit_word(ORR_SHIFTED[64] | (src << 16) | (XZR << 5) | dst)
       end
 
-      # str X{reg}, [slot]. The counterpart of #load_reg.
+      # str X{reg}, [slot]. The counterpart of #load_reg. The store itself is
+      # never skipped: the slot is the value's home, and nothing here knows
+      # whether a later branch reaches a reader by a path that goes nowhere near
+      # this register.
       def store_reg(reg, vreg)
+        refresh_slot_residency
+        promoted = @promoted[vreg]
+        if promoted
+          # The value's home is a register, so the store is a move into it —
+          # and nothing at all when the value was computed there. No slot is
+          # written and no residency can name a promotion register, so every
+          # entry in the table is still true.
+          emit_move_register(promoted, reg) unless reg == promoted
+          note_slots_undisturbed
+          return
+        end
+        if @transient[vreg]
+          # The value's only reader is the next instruction, which will find it
+          # here; the slot itself is never named again, so nothing is written.
+          note_slot_loaded(reg, vreg)
+          return
+        end
         store_frame_at(reg, slot_offset(vreg))
+        note_slot_stored(reg, vreg)
+      end
+
+      # Loads a two-operand instruction's operands into A and B.
+      #
+      # Which one is fetched first stops being arbitrary once #load_reg can
+      # reuse a resident value: if `b` is the value sitting in A, loading `a`
+      # there first would throw it away and force `b` back out of memory.
+      # Fetching B first instead keeps it, as a register move. A `commutative`
+      # op can do better still — swapping the two makes the resident operand A's,
+      # so nothing is moved at all — and is safe to swap precisely because the
+      # instruction's two register operands are interchangeable (which is not
+      # true of :sub, of the shifts, or of an ordering comparison).
+      def load_binary_operands(a, b, commutative: false)
+        refresh_slot_residency
+        if slot_resident_in?(A, b) && !slot_resident_in?(A, a)
+          if commutative
+            a, b = b, a
+          else
+            load_reg(B, b)
+          end
+        end
+        load_reg(A, a)
+        load_reg(B, b)
+      end
+
+      # --- naming promoted registers in place -------------------------------
+      #
+      # AArch64's data-processing instructions are three-address — Rd, Rn and
+      # Rm are independent five-bit fields naming any of the 31 general
+      # registers — and its loads and stores name their base register just as
+      # freely. So a promoted value is used *where it lives*, both as an
+      # operand and as a destination, and a promoted addition really is one
+      # instruction: `add x19, x19, x20`.
+      #
+      # That is the whole difference from the x86_64 backend, where the same
+      # step had to be split in two (put the value in a register, then teach
+      # each encoding to name it) because a two-address machine with one
+      # memory operand cannot express the second half for free. The measured
+      # consequence there was that promotion *alone* made code slower: every
+      # memory operand it removed became two register instructions. There is no
+      # such half-way state here.
+      #
+      # What is deliberately not widened is everything the ABI pins down: a
+      # call's arguments (x0..x7, v0..v7), its result (x0/v0), the atomic
+      # sequences' exclusive pair and the whole-object copy, whose loop
+      # advances the addresses it was handed and so must be given scratch
+      # registers of its own.
+
+      # The register an instruction should read `vreg` out of: the callee-saved
+      # register it was promoted into, a scratch register that already holds
+      # the value, or `scratch` with the value loaded into it.
+      #
+      # The middle case is what keeps a transient free. #load_binary_operands
+      # has to move one out of A when the *other* operand is on its way in
+      # there; where the other operand is promoted nothing is on its way in at
+      # all, so the value is read where its producer left it.
+      #
+      # Only A and B answer, though a residency may name any scratch register:
+      # those two are the ones nothing writes between the operands being named
+      # and the last instruction that reads them, while C is written in the
+      # middle of a remainder (it takes the quotient) and could equally be
+      # holding one of the values that remainder is about to read.
+      def operand_register(vreg, scratch)
+        promoted = @promoted[vreg]
+        return promoted if promoted
+
+        refresh_slot_residency
+        return A if slot_resident_in?(A, vreg)
+        return B if slot_resident_in?(B, vreg)
+
+        load_reg(scratch, vreg)
+        scratch
+      end
+
+      # The pair of registers a two-operand instruction should read, one per
+      # operand. A promoted operand is named where it is; the rest go through
+      # A and B as before, and when neither operand is promoted this is exactly
+      # #load_binary_operands — including its rescue of an operand still
+      # sitting in A, which only arises when something has to be loaded there.
+      #
+      # `commutative` is passed on for that rescue's sake alone. Nothing here
+      # needs the two exchanged the way the x86_64 backend does to reach an
+      # in-place form: this machine's destination field is free, so
+      # `sub x19, x20, x19` and `sub x19, x19, x20` cost the same one
+      # instruction and neither reads a register the other has overwritten —
+      # every source is read before Rd is written.
+      def binary_operand_registers(a, b, commutative: false)
+        promoted_a = @promoted[a]
+        promoted_b = @promoted[b]
+        return [promoted_a, promoted_b] if promoted_a && promoted_b
+        return [promoted_a, operand_register(b, B)] if promoted_a
+        return [operand_register(a, A), promoted_b] if promoted_b
+
+        load_binary_operands(a, b, commutative: commutative)
+        [A, B]
+      end
+
+      # Where an instruction that computes a general-register result should put
+      # it: the callee-saved register `dst` was promoted into, so the value is
+      # written home directly, or `scratch` for a value that lives in a slot.
+      def result_register(dst, scratch = A)
+        @promoted[dst] || scratch
+      end
+
+      # Completes such an instruction. A destination that lives in a slot still
+      # needs the store; a promoted `dst` *is* `reg`, so the value is already
+      # home and all that is left is telling the residency table what the bytes
+      # just emitted did to it.
+      #
+      # **The default answer is to throw the table away.** Bytes have been
+      # emitted since it was last known true — the caller's own instruction —
+      # and nothing here can see which registers they wrote. Keeping the table
+      # alive across them is #note_register_clobbered's exception, sound only
+      # when those bytes wrote `reg` and nothing else the table can name; that
+      # is a fact about one lowering's encoding, so the lowering is what states
+      # it, by passing `only_wrote: reg`. Anything that says nothing gets the
+      # safe reading, which costs an optimization and never correctness — and a
+      # lowering that later grows a second scratch write has to come back to its
+      # own claim rather than silently reviving a stale residency. That is the
+      # shape #emit_alloca and #emit_divmod were caught in: both wrote A or C on
+      # the way to the result and both revalidated the table on the way out.
+      #
+      # "Nothing else the table can name" means none of A, B, C or x0..x8, which
+      # are the registers #load_reg is ever asked to fill and so the only ones an
+      # entry can be keyed by. Writing sp, ADDR, x13..x15 or a promotion register
+      # does not disturb the table (see PROMOTION_REGISTERS and #load_reg), so it
+      # does not stop a caller from claiming the fast path.
+      def store_result(reg, dst, only_wrote: nil)
+        unless @promoted[dst]
+          store_reg(reg, dst) # which discards a stale table itself, before anything else
+          return
+        end
+
+        if only_wrote == reg
+          note_register_clobbered(reg)
+        else
+          refresh_slot_residency
+        end
       end
 
       # ldr X{reg}, [frame base + offset] for any fixed-frame offset, not just a
@@ -1294,13 +1781,49 @@ module Rubycc
       # extra instructions per operand and buy nothing, since ldr/str address a
       # V register with the same sp-relative form the X registers use — the sole
       # difference being the V bit that selects the register file.
+      #
+      # As with #load_reg, the move disappears when `reg` already holds this
+      # slot at this width and becomes a register-to-register `fmov` when
+      # another vector register does — which copies exactly the bits a reload
+      # would have produced at that width.
+      #
+      # Unlike #load_reg this asks nothing about promotion: a value the vector
+      # register file touches is refused promotion outright (IR::Promotion's
+      # VECTOR_OPS and VECTOR_KINDS), so every vreg reaching here has a slot.
       def load_fp(reg, vreg, size)
-        emit_fp_slot_access(reg, slot_offset(vreg), size, load: true)
+        refresh_slot_residency
+        return if slot_resident_in_vector?(reg, vreg, size)
+
+        source = vector_register_holding_slot(vreg, size)
+        if source
+          emit_word(FMOV_REG.fetch(size) | (source << 5) | reg)
+        else
+          emit_fp_slot_access(reg, slot_offset(vreg), size, load: true)
+        end
+        note_slot_loaded_to_vector(reg, vreg, size)
+      end
+
+      # Stages a floating instruction's operands into FA and FB. When the second
+      # was never written to its slot (a transient waiting in FA), it is rescued
+      # into FB before the first operand lands on top of it — the vector-file
+      # version of the ordering rule #load_binary_operands follows.
+      def load_float_operands(a, b, size)
+        refresh_slot_residency
+        load_fp(FB, b, size) if slot_resident_in_vector?(FA, b, size) &&
+                                !slot_resident_in_vector?(FA, a, size)
+        load_fp(FA, a, size)
+        load_fp(FB, b, size)
       end
 
       # str S{reg}/D{reg}, [slot]. The counterpart of #load_fp.
       def store_fp(reg, vreg, size)
+        refresh_slot_residency
+        if @transient[vreg]
+          note_slot_loaded_to_vector(reg, vreg, size)
+          return
+        end
         emit_fp_slot_access(reg, slot_offset(vreg), size, load: false)
+        note_slot_stored_from_vector(reg, vreg, size)
       end
 
       # The shared body of #load_fp / #store_fp. The unsigned-offset immediate is
@@ -1326,8 +1849,7 @@ module Rubycc
       # sign bit with an integer :xor, which the integer path already lowers.
       def emit_float_binary(inst, base_table)
         size = inst.size
-        load_fp(FA, inst.a, size)
-        load_fp(FB, inst.b, size)
+        load_float_operands(inst.a, inst.b, size)
         emit_word(base_table.fetch(size) | (FB << 16) | (FA << 5) | FA)
         store_fp(FA, inst.dst, size)
       end
@@ -1338,8 +1860,7 @@ module Rubycc
       # FLOAT_CONDITIONS). The `cset` is the 32-bit form, so the slot gets a
       # clean int with its upper half zeroed.
       def emit_float_comparison(dst, a, b, condition, size)
-        load_fp(FA, a, size)
-        load_fp(FB, b, size)
+        load_float_operands(a, b, size)
         emit_word(FCMP.fetch(size) | (FB << 16) | (FA << 5)) # fcmp FA, FB
         emit_cset(A, condition)
         store_reg(A, dst)
@@ -1416,10 +1937,11 @@ module Rubycc
       # finally loaded — but it does bind the symbol at link time, which is what
       # separates it from the GOT path below.
       def emit_symbol_address(dst, reloc)
+        target = result_register(dst)
         @relocations << reloc.merge(offset: @code.bytesize)
-        emit_word(0x90000000 | A)              # adrp A, <page of sym>
-        emit_add_imm(A, A, 0, shift12: false)  # add  A, A, #:lo12:sym
-        store_reg(A, dst)
+        emit_word(0x90000000 | target)              # adrp target, <page of sym>
+        emit_add_imm(target, target, 0, shift12: false) # add target, target, #:lo12:sym
+        store_result(target, dst, only_wrote: target) # the pair writes target alone
       end
 
       # :got_addr — the PIC form of the above. Under -fPIC a symbol this unit
@@ -1432,17 +1954,21 @@ module Rubycc
       # value that lands in the destination slot is a usable pointer either way
       # and every load or store through it is unchanged.
       def emit_got_address(dst, symbol)
+        target = result_register(dst)
         @relocations << { kind: :got, offset: @code.bytesize, symbol: symbol }
-        emit_word(0x90000000 | A)             # adrp A, <page of sym's GOT slot>
-        emit_word(0xF9400000 | (A << 5) | A)  # ldr  A, [A, #:got_lo12:sym]
-        store_reg(A, dst)
+        emit_word(0x90000000 | target)                       # adrp target, <page of GOT slot>
+        emit_word(0xF9400000 | (target << 5) | target)       # ldr  target, [target, #:got_lo12:]
+        store_result(target, dst, only_wrote: target) # the pair writes target alone
       end
 
       # :addr_of / :object_addr — compute a frame address (sp + offset) into a
       # scratch register and park it in the destination slot as a pointer value.
       def emit_slot_address_to(dst, offset)
-        emit_slot_address(A, offset)
-        store_reg(A, dst)
+        target = result_register(dst)
+        emit_slot_address(target, offset)
+        # However far the offset reaches, the composition builds it in target
+        # and reads only the frame base (#emit_base_address).
+        store_result(target, dst, only_wrote: target)
       end
 
       # Places (frame base) + offset (offset >= 0) into `reg`, the address of a
@@ -1602,6 +2128,12 @@ module Rubycc
       LSRV = { 32 => 0x1AC02400, 64 => 0x9AC02400 }.freeze
       ASRV = { 32 => 0x1AC02800, 64 => 0x9AC02800 }.freeze
 
+      # :scaled_add's element size -> the imm6 shift amount the shifted-register
+      # add above applies to its second operand (the shift type field stays 00,
+      # LSL). These four powers of two are exactly the set IR::Simplify fuses a
+      # subscript for.
+      SHIFTED_SCALES = { 1 => 0, 2 => 1, 4 => 2, 8 => 3 }.freeze
+
       # umulh Rd, Rn, Rm — a "data-processing (3 source)" instruction in the
       # same family as madd/msub, with op31 = 110 selecting the unsigned-high
       # variant and Ra fixed to xzr (31) in bits [14:10]. Only the 64-bit (sf=1)
@@ -1618,6 +2150,49 @@ module Rubycc
       RBIT = { 32 => 0x5AC00000, 64 => 0xDAC00000 }.freeze
       CLZ  = { 32 => 0x5AC01000, 64 => 0xDAC01000 }.freeze
 
+      # madd Rd, Rn, Rm, xzr — the plain multiply, which the architecture
+      # spells as a multiply-add whose addend is the zero register. Both :mul
+      # and the population count's last stage need it, at either width.
+      MADD_ZERO = { 32 => 0x1B007C00, 64 => 0x9B007C00 }.freeze
+
+      # "Bitfield" (UBFM) and "Logical (immediate)", the two families the
+      # population count expands into:
+      #   sf(31) opc(30:29) 100110(28:23) N(22) immr(21:16) imms(15:10) Rn Rd
+      #                                                      -- UBFM, opc = 10
+      #   sf(31) opc(30:29) 100100(28:23) N(22) immr(21:16) imms(15:10) Rn Rd
+      #                                             -- AND/ORR immediate, opc =
+      #                                                00 (AND) and 01 (ORR)
+      # A shift by an immediate has no opcode of its own: `lsr Rd, Rn, #s` is
+      # UBFM with immr = s and imms = the register's top bit (31 or 63), which
+      # extracts the bits from s upwards and right-aligns them. Its 64-bit form
+      # sets N = 1 (a 64-bit bitfield), which is the 0x400000 in that base; the
+      # two logical bases leave N = 0, every pattern the count needs being
+      # shorter than 32 bits (below).
+      LSR_IMM = { 32 => 0x53000000, 64 => 0xD3400000 }.freeze
+      AND_IMM = { 32 => 0x12000000, 64 => 0x92000000 }.freeze
+      MOV_IMM = { 32 => 0x32000000, 64 => 0xB2000000 }.freeze # orr Rd, xzr, #imm
+
+      # The masks and the multiplier of the population count, as their imms
+      # fields. A logical immediate is a pattern of `esize` bits — 2, 4, 8, 16,
+      # 32 or 64 — repeated to fill the register, holding a run of ones that
+      # immr then rotates; N:imms names the element size and the run length
+      # together, by how many leading ones imms carries:
+      #
+      #   0b11110s  esize 2,  s + 1 ones   0x5555...  s = 0: one 1 per pair
+      #   0b1110ss  esize 4,  s + 1 ones   0x3333...  s = 1: two 1s per nibble
+      #   0b110sss  esize 8,  s + 1 ones   0x0F0F...  s = 3: four 1s per byte
+      #                                    0x0101...  s = 0: one 1 per byte
+      #
+      # All four runs start at bit 0, so immr is 0 for each and the same field
+      # serves either width (N = 0 in both, the pattern being shorter than 32
+      # bits). This is why the expansion needs no movz/movk anywhere: the
+      # constants it works with are precisely the repeating kind this encoding
+      # was made for.
+      POPCOUNT_PAIR_IMMS = 0b111100
+      POPCOUNT_NIBBLE_IMMS = 0b111001
+      POPCOUNT_BYTE_IMMS = 0b110011
+      POPCOUNT_ONES_IMMS = 0b110000
+
       # "Add/subtract (extended register)", 64-bit, with option = 011 (UXTX, the
       # identity extension of an X operand) and no shift:
       #   sf(31)=1 op(30) S(29)=0 01011(28:24) opt(23:22)=00 1(21) Rm(20:16)
@@ -1698,6 +2273,19 @@ module Rubycc
         8 => { load: 0xFD400000, store: 0xFD000000 }
       }.freeze
 
+      # The same form at the 128-bit width, which the encoding reaches through
+      # the opc field rather than the size field: size = 00 with opc = 11 is
+      # "ldr q, [Xn]". Only the zero-offset load is needed — a quad-precision
+      # argument is read once from the object the generator built it in — so
+      # this is the bare instruction with an empty imm12.
+      LDR_Q = 0x3DC00000
+
+      # "fmov Sd, Sn" / "fmov Dd, Dn", the vector-file register move, keyed by
+      # the value's width — which is also the encoding's type field (00 single,
+      # 01 double). Used where a floating value wanted in one scratch register
+      # is already in the other.
+      FMOV_REG = { 4 => 0x1E204000, 8 => 0x1E604000 }.freeze
+
       # The same form with V = 0, the general-purpose register file, at each of
       # the four access widths. The narrow loads are the zero-extending ones
       # (ldrb/ldrh rather than ldrsb/ldrsh): both a whole-object copy and an