@asmlift/core 0.9.0 → 0.11.0

package/src/raise/paramwidth.ts

@@ -122,11 +122,46 @@
 // pool load in the benchmark's agbcc references (extscale.ts's header has the census). Only the
 // fold's own extensions are judged this way; a plain cast's extension sits at its right shift,
 // where its position says nothing about its `lsl`'s.
-import { type Fn, type Op, type Value, replaceAllUsesWith, successorsOf } from '../ir/core';
+//
+// EVERY READER AN EXTENSION, the `/narrow-param` lift variation (`narrowExtendedParams`). agbcc does
+// not always widen a narrow declared parameter in the prologue: where every use of it is itself an
+// extension it widens at each use instead, and then the object carries no declaration to read.
+// `pokeemerald:LoadMonInfo` declares `s16 partyId` and reads it as `lsl #16` behind a pool load, an
+// `asr #14` for the scaled index and an `lsr #16` for a `u16` argument, so `raw-reader` refuses it
+// (two readers), and with that ablated `not-prologue` (behind the pool load) and `fused-behind-pool`.
+// Declared `s16` and read `a << 2` and `(u16)a`, or declared `s32` and read `(s16)a << 2` and
+// `(u16)a`, the two are one object in isolation and differ only through the register allocation
+// around them (a sibling `u8` parameter's prologue copy), so nothing in the asm decides it and the
+// differ referees. Declaring the parameter at width w changes no reader's value when every reader is
+// an extension to w' ≤ w: ext_w'(trunc_w x) = ext_w'(x). The width is the widest reader's, its
+// signedness the caller's declaration where that spells one and the first reader of that width's
+// otherwise; each reader of exactly that type becomes the parameter itself. Where no prototype
+// declares the width, the candidate loses every score tie (rank.ts `guessesParamWidth`): without
+// a sibling to separate them the two declarations are often one object, and the narrow one costs
+// each prototyped caller the truncation `proto-width` is about.
+//
+// A LOW MASK IS AN EXTENSION READER. mwcc reads a `u16` parameter as `clrlwi rD,rS,16`, which the
+// PPC frontend lifts as `and rS, 0xffff` — as it must, since the same instruction is every other
+// mask too — so a parameter mwcc reads only through it reaches this pass with no `zext` reader at
+// all. `and v, 0xffff` is `zext v {width=16}` value for value, so it counts as one (`extensionOf`),
+// unsigned, and becomes the parameter itself when the parameter is declared `u16`. The two
+// declarations are not one object there: over a `long *a`, mwcc_242_81 compiles
+// `*a = *a & ~b; *a = *a | c;` to `not; and` when `b` and `c` are `unsigned short`, and to `andc` when
+// they are `unsigned long` and the body masks them, at `-O4,p` and at Mario Party 4's unit flags.
+//
+// The rule is about the IR, not the ISA, so it reaches every frontend that lifts a mask as `and`.
+// IDO reads a declared `u16`/`u8` parameter as `sw a0,0(sp); andi a0,a0,0xffff`, and frontend/mips.ts
+// lifts the `andi` as that mask too: `s32 f(u16 a, s32 b){ return a + b; }` scores 3 wide and 0
+// narrowed. The witness gates still decide whether IDO may narrow: a mask applied in place with no
+// home store (`a = a & 0xffff`) is refused by `unhomed-param`. agbcc spells a mask as `and` with a
+// pooled `0xffff` or a `mov #0xff`, so the variation is offered there too, and loses, because agbcc
+// extends a declared narrow parameter with a shift pair: `s32 f(u32 a, s32 b){ return (a & 0xffff)
+// + b; }` scores 0 wide and 6 narrowed.
+import { type Fn, type Op, type Value, defOpMap, replaceAllUsesWith, successorsOf } from '../ir/core';
 import { CAST_WIDTHS, MATERIALIZING_OPS } from '../ir/opcodes';
 import { T } from '../ir/types';
 import { type Gate, firstRejection } from '../l3/gates';
-import { type FnProto, declaredWidth } from '../proto';
+import { type FnProto, declaredSigned, declaredWidth } from '../proto';
 import type { NarrowParamWitness } from '../target';
 
 /** What the gates below judge: one entry parameter and the extension that reads it. */
@@ -141,6 +176,8 @@ export interface NarrowParamCandidate {
   inPrologue: boolean;
   /** reads of the RAW parameter anywhere in the function */
   uses: number;
+  /** reads of it that are not an extension to `width` or narrower */
+  wideReaders: number;
   /** the width the caller's own prototype declares for this parameter, if it declares one */
   declared: number | undefined;
   /** the extension is one raise/extscale.ts re-split from a fused pair whose `shl` the machine ran
@@ -192,14 +229,14 @@ export const PARAM_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
   },
   {
     id: 'not-prologue',
-    why: 'an extension behind body code is where the SOURCE wrote the cast — and on a schedule that interleaves them, the refusing answer',
+    why: 'an extension behind body code may be the source writing the cast, and nothing in the fan referees a narrow signature against the wide one',
     sound: true,
     guardedBy: 'param-width.test.ts: an extension behind a nullary call is body code',
     rejects: (c) => !c.inPrologue,
   },
   {
     id: 'no-declaration-witness',
-    why: 'this compiler spells a narrow declaration and a body cast the same way, so the asm decides nothing',
+    why: 'the compiler spells a narrow declaration and a body cast the same way, so the assembly decides nothing',
     sound: true,
     guardedBy:
       'param-width.test.ts: a compiler whose two spellings are one object refuses the narrowing, measured or not',
@@ -207,14 +244,14 @@ export const PARAM_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
   },
   {
     id: 'unhomed-param',
-    why: 'this compiler homes a narrow DECLARED parameter, so the absent home store proves the declaration was wide',
+    why: 'the compiler stores a narrow declared parameter to the stack on entry, so one it did not store was declared wide',
     sound: true,
     guardedBy: 'param-width.test.ts: a homing compiler refuses the parameter it did not home',
     rejects: (c) => c.witness === 'home-store-and-in-place' && !c.homed,
   },
   {
     id: 'widened-elsewhere',
-    why: 'this compiler widens a narrow DECLARED parameter in the argument register itself, so a widening that lands in a scratch register is not a declaration',
+    why: 'the compiler widens a narrow declared parameter in its own argument register, so a widening into another register is no declaration',
     sound: true,
     guardedBy:
       'param-width.test.ts: \u2026and refuses one it homed but widened SOMEWHERE ELSE \u2014 that is a 64-bit half',
@@ -229,6 +266,96 @@ export const PARAM_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
   },
 ];
 
+const byId = (id: string): Gate<NarrowParamCandidate> => PARAM_WIDTH_GATES.find((g) => g.id === id)!;
+
+/** `/narrow-param`'s table (`narrowExtendedParams`): a second CONSUMER of the default's rules, not an
+ *  ablation of its table, so it is built by id rather than through `ablateHeuristic`. It leaves out
+ *  the three rules that read the prologue's evidence and adds the one that reads the readers.
+ *  `raw-reader` gives way to `wide-reader`, which is sound for the same reason it is: every reader
+ *  is an extension the pass rewrites. `not-prologue` and `fused-behind-pool` read fields this pass
+ *  does not observe (it fills `inPrologue` and `fusedBehindPool` false). Their `sound` is a claim
+ *  about the DEFAULT, whose narrowing reaches every candidate with no wide sibling, so a wrong width
+ *  there is a signature nothing referees and every prototyped caller pays for. A /narrow-param
+ *  candidate keeps the wide default beside it and loses every tie unless a prototype declares the
+ *  width (rank.ts `guessesParamWidth`), so for it the same question is refereed: a rule sound for a
+ *  declaration is a heuristic for a generated candidate (l3/gates.ts, `ORDER_SHAPE_GATES`).
+ *  param-width.test.ts holds the set left out to exactly these three, so a sound rule added to the
+ *  default is either added here or argued out by name. */
+export const PARAM_READER_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
+  byId('entry-is-join'),
+  byId('param-typed'),
+  byId('cast-width'),
+  {
+    id: 'wide-reader',
+    why: 'a reader that is not an extension to this width or narrower reads bits a narrow declaration drops',
+    sound: true,
+    guardedBy: 'param-width.test.ts: a reader that is no narrow extension keeps the parameter wide',
+    rejects: (c) => c.wideReaders !== 0,
+  },
+  byId('proto-width'),
+  byId('no-declaration-witness'),
+  byId('unhomed-param'),
+  byId('widened-elsewhere'),
+];
+
+const isExtension = (op: Op): boolean => op.opcode === 'sext' || op.opcode === 'zext';
+
+/** The unsigned extension a MASK spells: `and v, 0xff` is `zext v {width=8}` and `and v, 0xffff` is
+ *  `zext v {width=16}`, value for value. */
+const MASK_WIDTHS: ReadonlyMap<number, number> = new Map([
+  [0xff, 8],
+  [0xffff, 16],
+]);
+
+/** What `op` reads `v` as, when it reads it ONLY through an extension: its width and signedness.
+ *  An `and` with a low mask counts, because that is the extension's spelling where the machine
+ *  has no extension of its own to lift: PPC's `clrlwi rD,rS,16` and MIPS's `andi rD,rS,0xffff`
+ *  (each frontend lifts it as the mask, and must, because every other mask is the same instruction).
+ *  Null for any other reader, including an extension or a mask of something besides `v` alone. */
+function extensionOf(op: Op, v: Value, defs: ReadonlyMap<Value, Op>): { width: number; signed: boolean } | null {
+  if (isExtension(op)) {
+    return op.operands.length === 1 && op.operands[0] === v
+      ? { width: op.attrs.width as number, signed: op.opcode === 'sext' }
+      : null;
+  }
+  if (op.opcode !== 'and' || op.operands.length !== 2 || !op.operands.includes(v)) {
+    return null;
+  }
+  const mask = defs.get(op.operands[op.operands[0] === v ? 1 : 0]);
+  const width = mask?.opcode === 'const' ? MASK_WIDTHS.get(mask.attrs.value as number) : undefined;
+  return width === undefined ? null : { width, signed: false };
+}
+
+/** The ops that read `v` and the number of branch arguments that pass it, anywhere in `fn`. */
+function readersOf(fn: Fn, v: Value): { ops: Op[]; edges: number } {
+  const ops: Op[] = [];
+  let edges = 0;
+  for (const b of fn.blocks) {
+    for (const op of b.ops) {
+      if (op.operands.includes(v)) {
+        ops.push(op);
+      }
+      for (const s of op.successors) {
+        edges += s.args.filter((a) => a === v).length;
+      }
+    }
+  }
+  return { ops, edges };
+}
+
+/** How many reads of `v` are not an extension of `v` alone (`extensionOf`) to `width` or narrower. */
+function wideReaderCount(fn: Fn, v: Value, width: number, defs: ReadonlyMap<Value, Op>): number {
+  const { ops, edges } = readersOf(fn, v);
+  return (
+    edges +
+    ops.reduce(
+      (n, op) =>
+        n + ((extensionOf(op, v, defs)?.width ?? Infinity) <= width ? 0 : op.operands.filter((o) => o === v).length),
+      0,
+    )
+  );
+}
+
 /** How many times `v` is read anywhere in `fn` — op operands and branch arguments alike. */
 function useCount(fn: Fn, v: Value): number {
   let n = 0;
@@ -282,6 +409,7 @@ export function narrowEntryParams(
     }
     prologue.add(op);
   }
+  const defs = defOpMap(fn);
   let narrowed = 0;
   for (const op of [...entry.ops]) {
     if (op.opcode !== 'sext' && op.opcode !== 'zext') {
@@ -298,6 +426,7 @@ export function narrowEntryParams(
       entryIsJoin,
       inPrologue: prologue.has(op),
       uses: useCount(fn, p),
+      wideReaders: wideReaderCount(fn, p, width, defs),
       declared: declared[entry.params.indexOf(p)],
       fusedBehindPool: fusedBehindPool.has(op),
       witness,
@@ -314,3 +443,64 @@ export function narrowEntryParams(
   }
   return narrowed;
 }
+
+/** `/narrow-param`: type each entry parameter every reader of which is an extension of it at the
+ *  widest reader's width, and drop the readers of exactly that type (EVERY READER AN EXTENSION).
+ *  `apply: false` changes nothing and reports what it would narrow. Returns each parameter it
+ *  narrowed. */
+export function narrowExtendedParams(
+  fn: Fn,
+  witness: NarrowParamWitness,
+  self?: FnProto,
+  gates: readonly Gate<NarrowParamCandidate>[] = PARAM_READER_WIDTH_GATES,
+  apply = true,
+): NarrowParamCandidate[] {
+  const evidence = fn.paramEvidence;
+  const entry = fn.blocks[0];
+  const declaredTypes = Array.isArray(self?.params) ? self.params : [];
+  const entryIsJoin = fn.blocks.some((b) => successorsOf(b).includes(entry));
+  const narrowed: NarrowParamCandidate[] = [];
+  const defs = defOpMap(fn);
+  for (const [i, p] of entry.params.entries()) {
+    const extensions = readersOf(fn, p).ops.flatMap((op) => {
+      const ext = extensionOf(op, p, defs);
+      return ext === null ? [] : [{ op, ...ext }];
+    });
+    if (extensions.length === 0) {
+      continue;
+    }
+    const width = Math.max(...extensions.map((e) => e.width));
+    const first = extensions.find((e) => e.width === width)!;
+    const declared = declaredTypes[i];
+    const c: NarrowParamCandidate = {
+      param: p,
+      width,
+      entryIsJoin,
+      inPrologue: false,
+      uses: useCount(fn, p),
+      wideReaders: wideReaderCount(fn, p, width, defs),
+      declared: declared === undefined ? undefined : declaredWidth(declared),
+      fusedBehindPool: false,
+      witness,
+      homed: evidence?.get(p)?.deadHome ?? false,
+      selfRedefined: evidence?.get(p)?.selfRedefined ?? false,
+    };
+    if (firstRejection(gates, c) !== null) {
+      continue;
+    }
+    narrowed.push(c);
+    if (!apply) {
+      continue;
+    }
+    const signed = (declared === undefined ? undefined : declaredSigned(declared)) ?? first.signed;
+    p.type = T.int(width, signed);
+    for (const { op, ...ext } of extensions) {
+      if (ext.width === width && ext.signed === signed) {
+        replaceAllUsesWith(fn, op.results[0], p);
+        const home = fn.blocks.find((b) => b.ops.includes(op))!;
+        home.ops.splice(home.ops.indexOf(op), 1);
+      }
+    }
+  }
+  return narrowed;
+}

package/src/raise/pre-recovery.ts

@@ -12,12 +12,13 @@
 // in the driver.
 import { Block, Fn } from '../ir/core';
 import { simplifyTrivialPhis } from '../ir/simplify';
-import { dce } from '../pattern/engine';
-import type { FnProto } from '../proto';
+import { NOT_FCMP_PATTERNS, applyPattern, dce } from '../pattern/engine';
+import { type FnProto, declaresVoidReturn } from '../proto';
 import type { TargetDescription } from '../target';
+import { readDeclaredMembers } from './aggmembers';
 import { recognizeArrays } from './arrays';
 import { recognizeConsts } from './const';
-import { recognizeDivPow2 } from './divpow2';
+import { recognizeDivPow2, recognizeSremPow2 } from './divpow2';
 import {
   type PoolOrder,
   type ScaleRecord,
@@ -33,12 +34,13 @@ import { recognizeMagicDivision } from './magicdiv';
 import { recognizeMemberArrays } from './memberarrays';
 import { rerootNarrowReads } from './narrow';
 import { type MergeShape, mergeShapes, narrowBlockLocals } from './narrowlocal';
-import { fuseParamPairs } from './pairparams';
-import { PARAM_WIDTH_GATES, narrowEntryParams } from './paramwidth';
+import { fuseParamPairs, refuseSplitWideParams } from './pairparams';
+import { PARAM_READER_WIDTH_GATES, PARAM_WIDTH_GATES, narrowEntryParams, narrowExtendedParams } from './paramwidth';
 import { type BranchShortCircuitOptions, recognizeBranchShortCircuit, recognizeShortCircuit } from './shortcircuit';
 import { recognizeSoftDiv } from './softdiv';
+import { seedNarrowStoreSigns } from './storesign';
 import { recognizeStructArrays } from './struct-arrays';
-import { recognizeStructs } from './structs';
+import { type DisplacedOptions, foldDisplacedAccesses, recognizeStructs } from './structs';
 import { TRUNC_LOAD_GATES, foldTruncatedLoads } from './truncload';
 import { recognizeWideHelpers, refuseUnmodelledHelpers } from './widehelpers';
 
@@ -49,6 +51,19 @@ import { recognizeWideHelpers, refuseUnmodelledHelpers } from './widehelpers';
 export interface PreRecoveryOptions {
   /** raise/shortcircuit.ts `recognizeBranchShortCircuit` — the connective-vs-comparison-tree question. */
   shortCircuit?: BranchShortCircuitOptions;
+  /** raise/paramwidth.ts `narrowExtendedParams` — the `/narrow-param` lift. */
+  paramWidth?: ParamWidthOptions;
+  /** raise/structs.ts `foldDisplacedAccesses` — the `/far-cast` lift. */
+  displaced?: DisplacedOptions;
+}
+
+export interface ParamWidthOptions {
+  /** also narrow a parameter every reader of which is an extension (`/narrow-param`) */
+  extendedReaders?: boolean;
+  /** called when a lift without `extendedReaders` has a parameter that would narrow with it */
+  onExtendedReaders?: () => void;
+  /** called when the `extendedReaders` lift narrows a parameter whose width no prototype declares */
+  onUndeclaredWidth?: () => void;
 }
 
 /** CFG facts read off the function ONCE, before the first pass below rewrites it.
@@ -94,10 +109,10 @@ export interface PreRecoveryPass {
 }
 
 /** THE ordered pre-recovery pass list — the single source of truth shared by pipeline / rank / report.
- *  address-numbering → const-materialize → magic-division → pow2-division → soft-division →
- *  scaled-extension → array-legalize → truncated-load → struct-array → member-array → struct-pointer →
- *  short-circuit → branch-short-circuit → narrow-reads → narrow-local → parameter-width →
- *  scaled-extension-restore.
+ *  address-numbering → const-materialize → magic-division → pow2-division → pow2-remainder →
+ *  soft-division → scaled-extension → store-sign → array-legalize → truncated-load → struct-array →
+ *  member-array → struct-pointer → short-circuit → branch-short-circuit → narrow-reads →
+ *  narrow-local → parameter-width → scaled-extension-restore.
  *  See each recognizer's file for the rationale. */
 export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // FIRST: collapsing duplicate address definitions removes block params every later recognizer
@@ -126,6 +141,9 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // ending in `br`. Verified by running it before and after each of them — same result. It sits
   // beside magicdiv so that a reader looking for division recovery finds both together.
   { id: 'divpow2', run: recognizeDivPow2, dce: true },
+  // …and its remainder sibling, IDO's two-test diamond (raise/divpow2.ts `recognizeSremPow2`): no
+  // other pass reads a three-edge merge whose last arm takes a power of two off the mask.
+  { id: 'srempow2', run: recognizeSremPow2, dce: true },
   {
     id: 'softdiv',
     run: (fn, _self, _opts, target) => recognizeSoftDiv(fn, target),
@@ -137,9 +155,19 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // from `widehelpers` because it re-types words as floats — a helper's, a parameter a callee
   // declares `double`, a literal — with its own closed-set refusal, and a single-precision helper
   // is a float helper that is not wide.
+  // A fold re-runs the float compares' negation idioms (pattern/engine.ts `NOT_FCMP_PATTERNS`),
+  // which ran before there was a float compare to negate.
   {
     id: 'floathelpers',
-    run: (fn, _self, _opts, target) => recognizeFloatHelpers(fn, target),
+    run: (fn, _self, _opts, target) => {
+      if (!recognizeFloatHelpers(fn, target)) {
+        return false;
+      }
+      if (NOT_FCMP_PATTERNS.reduce((n, p) => n + applyPattern(fn, p, target), 0) > 0) {
+        dce(fn);
+      }
+      return true;
+    },
     dce: false,
   },
   // NOT gated, where `softdiv` is, and the asymmetry is the argument: a soft DIVISION is
@@ -166,7 +194,17 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // AFTER it, because the `concat`s it reads are the ones the frontend built at the call the pass
   // above has just rewritten — and because a pair the rewrite refused is a pair this must not fuse
   // into a signature either.
-  { id: 'pairparams', run: (fn) => fuseParamPairs(fn), dce: false },
+  // It also refuses a declared 64-bit parameter the lift left in halves, which is the one place the
+  // fusion's answer is final (`refuseSplitWideParams`).
+  {
+    id: 'pairparams',
+    run: (fn, self) => {
+      const fused = fuseParamPairs(fn);
+      refuseSplitWideParams(fn, self);
+      return fused;
+    },
+    dce: false,
+  },
   // AFTER `const`, which folds a shift pair over a constant to the constant it computes, and BEFORE
   // the three array recognizers, whose input this pass produces: `shl(ext(x), k)` is an element
   // scale they legalize and the fused pair is not. `dce: true` — the `shl` a fold leaves readerless.
@@ -178,6 +216,11 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
     dce: true,
     gate: foldsShiftPairCasts,
   },
+  // BEFORE `arrays` and `struct-arrays`, which carry the stamp into the `astore` they make (and
+  // struct-arrays types a field only ever stored from it), and before `structs`, which does the
+  // same for a constant-offset field. `member-arrays` overwrites an astore's `signed` with its
+  // member's (raise/memberarrays.ts says why). `dce: false` — it changes no op.
+  { id: 'storesign', run: (fn, _self, _opts, target) => seedNarrowStoreSigns(fn, target), dce: false },
   { id: 'arrays', run: recognizeArrays, dce: true },
   // BEFORE the three struct synthesizers, and that is what the pass is FOR: `raise/structs.ts` is
   // the pass that refuses one offset read at two widths, and it can only see one width per offset
@@ -207,7 +250,11 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // struct-arrays AFTER arrays (scalar stride==width shapes are claimed first — see the
   // discriminator note in raise/struct-arrays.ts) and BEFORE structs (an element's field
   // accesses must not be re-derived as constant-offset struct-pointer accesses).
-  { id: 'struct-arrays', run: recognizeStructArrays, dce: true },
+  {
+    id: 'struct-arrays',
+    run: (fn, _self, _opts, target) => recognizeStructArrays(fn, target.compilerBehaviors.aggregateBoundary),
+    dce: true,
+  },
   // member-arrays AFTER struct-arrays, and that order IS load-bearing — but the order alone does
   // not deconflict them, because the two passes claim different VALUES: struct-arrays types the
   // materialized `add(P, K)` and member-arrays groups on `P`, so `P->tbl[i].f` reaches both. What
@@ -217,11 +264,28 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // sits there so the three struct synthesizers read in the order of their evidence. `narrowlocal`
   // reads the carrier's extension chain rather than any address, so nothing here orders against it.
   { id: 'member-arrays', run: (fn) => recognizeMemberArrays(fn), dce: true },
+  // RIGHT BEFORE `structs`, whose input it widens: an access through a materialised `add(p, K)`
+  // joins `p`'s access set at `off + K`. After `truncload`, `struct-arrays` and `member-arrays`, so
+  // none of them sees a folded access, and a displacement either of the last two typed is refused.
+  // `dce: true` — the `add` chain a fold leaves readerless. `opts.displaced` is `/far-cast`.
+  {
+    id: 'displaced',
+    run: (fn, self, opts, target) =>
+      foldDisplacedAccesses(fn, opts.displaced, target.compilerBehaviors, declaresVoidReturn(self)),
+    dce: true,
+  },
   {
     id: 'structs',
     run: (fn, _self, _opts, target) => recognizeStructs(fn, target.compilerBehaviors.aggregateBoundary),
     dce: false,
   },
+  // AFTER `structs`, which synthesizes layouts for bases it can type and never for a frame object
+  // the audit already typed as a declared struct — the only base this reads.
+  {
+    id: 'aggmembers',
+    run: (fn, _self, _opts, target) => readDeclaredMembers(fn, target.capabilities.endianness === 'little'),
+    dce: true,
+  },
   { id: 'shortcircuit', run: recognizeShortCircuit, dce: true },
   // The control-flow sibling, value form FIRST.
   //
@@ -278,16 +342,13 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // The `target` argument is read by TWO of this pass's gates, and unlike `narrowlocal`'s single
   // conjunct it is not a tuning knob: `narrowParamWitness` says WHICH fact in the object settles a
   // parameter's declared width on this compiler, and a target that names none refuses outright.
+  //
+  // `opts.paramWidth` adds the `/narrow-param` lift after it, over the parameters it left wide
+  // (raise/paramwidth.ts, EVERY READER AN EXTENSION), or asks whether that lift would narrow any.
   {
     id: 'paramwidth',
-    run: (fn, self, _opts, target, lifted) =>
-      narrowEntryParams(
-        fn,
-        target.compilerBehaviors.narrowParamWitness ?? 'none',
-        self,
-        PARAM_WIDTH_GATES,
-        lifted.scales.behindPool,
-      ),
+    run: (fn, self, opts, target, lifted) =>
+      narrowParamWidths(fn, self, opts.paramWidth, target, lifted, PARAM_WIDTH_GATES, PARAM_READER_WIDTH_GATES),
     dce: false,
   },
   // LAST, after every pass that can CLAIM what `extscale` exposed — the two width passes above take
@@ -302,6 +363,31 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   },
 ];
 
+/** The `paramwidth` entry's body, taking both of its tables, so a census can pass wrapped ones. */
+export function narrowParamWidths(
+  fn: Fn,
+  self: FnProto | undefined,
+  opts: ParamWidthOptions | undefined,
+  target: TargetDescription,
+  lifted: PreRecoveryFacts,
+  prologueGates: typeof PARAM_WIDTH_GATES,
+  readerGates: typeof PARAM_READER_WIDTH_GATES,
+): number {
+  const witness = target.compilerBehaviors.narrowParamWitness ?? 'none';
+  const n = narrowEntryParams(fn, witness, self, prologueGates, lifted.scales.behindPool);
+  if (opts?.extendedReaders) {
+    const lifted = narrowExtendedParams(fn, witness, self, readerGates);
+    if (lifted.some((c) => c.declared === undefined)) {
+      opts.onUndeclaredWidth?.();
+    }
+    return n + lifted.length;
+  }
+  if (opts?.onExtendedReaders && narrowExtendedParams(fn, witness, self, readerGates, false).length > 0) {
+    opts.onExtendedReaders();
+  }
+  return n;
+}
+
 /** Run the pre-recovery passes in order. For each pass whose gate passes and that CHANGES the IR, run
  *  `dce` when the pass declares it, then invoke `afterPass(pass, result)` (the caller's verify/trace
  *  hook), if given.

package/src/raise/pre-structure.ts

@@ -0,0 +1,73 @@
+// asmlift — the CFG folds that run last before structuring, after return-sinking: the loop folds
+// rewrite a loop's edges into a shape a loop spelling owns, and the last one takes a parameter off a
+// switch's join (a case fallen into, or the switch's exit). Each is its own stage in the trace, so a
+// re-spelled function names the fold that moved it.
+//
+// A MUTABLE RECORD, not a sum of calls, for the reason `PRE_RECOVERY_PASSES` is one: it is the
+// caller-side seam a process outside core swaps to census a fold's gate table (`pnpm bench gates`,
+// apps/benchmark/src/run/gate-census.ts). A static import binding cannot be swapped.
+import { Fn } from '../ir/core';
+import { foldEmptyLatches, foldExitTrampolines } from './latch';
+import { rerollPeeledTests } from './reroll';
+import { recomputeDerivedParams, recomputeRedundantParams } from './unpre';
+import { unthreadLoopExits } from './unthread';
+
+export interface PreStructureFold {
+  /** stable; the trace stage is `stage:<id>` */
+  readonly id: string;
+  /** the trace stage's title */
+  readonly title: string;
+  /** how many rewrites the fold made; 0 leaves the function as it was */
+  run: (fn: Fn) => number;
+}
+
+/** THE ORDER CARRIES LOAD, and each fold runs ONCE: there is no fixpoint across folds, so a fold
+ *  that exposes another's shape must run before it. The re-roll needs the latch's way out and the
+ *  entry's to reach ONE block, and on `synthetic:break_value:agbcc` they reach it only after the
+ *  trampoline fold has spliced out the empty block the break leaves through; ahead of it, the re-roll
+ *  declines and the loop with it ('unrecovered back-edge'). `reroll.test.ts` pins it by lifting that
+ *  row's pre-fold IR through this record. The latch fold comes first for the same reason in kind:
+ *  the folds after it read the back edge it leaves. The un-threading reads loop exits, which the
+ *  trampoline fold reshapes, and sits ahead of the re-roll so a loop it gives one way out is one
+ *  the re-roll may then read. That place is a precaution: `synthetic:thrbreak:agbcc` and the
+ *  pointer walk unthread.ts names lift the same C wherever the fold sits, so `unthread.test.ts`
+ *  pins its position, not a lift that needs it. The two parameter recomputes run last, the
+ *  derived one (`derivedparam`) just ahead of `unpre`, because that is the one place each was
+ *  measured, and `unpre.test.ts` pins the places, not a lift that needs them. Both MOVE EDGES: a
+ *  block either one's orphan sweep leaves holding only its `br` is threaded out of the CFG
+ *  (`threadEmptied`, raise/unpre.ts), its in-edges retargeted and the block removed, so `unpre`
+ *  reads the CFG `derivedparam` left, and a loop fold moved after either would read a reshaped one
+ *  it was never measured on. */
+export const PRE_STRUCTURE_FOLDS: PreStructureFold[] = [
+  {
+    id: 'latchfold',
+    title: 'Empty-latch folding (a latch that only jumps back becomes its back edge)',
+    run: (fn) => foldEmptyLatches(fn),
+  },
+  {
+    id: 'trampoline',
+    title: 'Exit-trampoline folding (an empty block on a loop exit becomes that edge)',
+    run: (fn) => foldExitTrampolines(fn),
+  },
+  {
+    id: 'unthread',
+    title: 'Loop-exit un-threading (an exit jump-threaded past the re-test after the loop goes back to it)',
+    run: (fn) => unthreadLoopExits(fn),
+  },
+  {
+    id: 'reroll',
+    title: 'Peeled-test re-rolling (a loop test peeled into the entry becomes the first iteration)',
+    run: (fn) => rerollPeeledTests(fn),
+  },
+  {
+    id: 'derivedparam',
+    title:
+      "Derived-parameter recomputing (a parameter every edge hands one pure op of a sibling's argument becomes that op)",
+    run: (fn) => recomputeDerivedParams(fn),
+  },
+  {
+    id: 'unpre',
+    title: 'Redundant-parameter recomputing (a parameter every edge hands one pure expression becomes it)',
+    run: (fn) => recomputeRedundantParams(fn),
+  },
+];

package/src/raise/recover.ts

@@ -2,8 +2,9 @@
 // (signed comparisons/divisions ⇒ signed operands), types memory-access bases as pointers,
 // propagates pointer-ness across the SSA, then defaults the rest to s32. Emits recovered
 // types onto the SSA values in place.
-import { Fn, Value, defOpMap } from '../ir/core';
+import { Fn, RETURNS_OBJECT, Value, defOpMap } from '../ir/core';
 import { IrType, T, scalarTypeForAccess, typeEquals } from '../ir/types';
+import { storeSigned } from './storesign';
 
 const SIGNED_CMP = new Set(['icmp_slt', 'icmp_sle', 'icmp_sgt', 'icmp_sge']);
 const UNSIGNED_CMP = new Set(['icmp_ult', 'icmp_ule', 'icmp_ugt', 'icmp_uge']);
@@ -87,7 +88,7 @@ function typeDerefBases(fn: Fn): void {
           break;
         case 'store':
           width = op.attrs.width as number;
-          signed = width === 4;
+          signed = storeSigned(op, width);
           break;
         case 'aload':
           width = op.attrs.elemSize as number;
@@ -95,9 +96,7 @@ function typeDerefBases(fn: Fn): void {
           break;
         case 'astore':
           width = op.attrs.elemSize as number;
-          // an astore carries no signedness of its own unless a recovery recorded the member's;
-          // structure.ts reads it by the same rule
-          signed = (op.attrs.signed as boolean | undefined) ?? width === 4;
+          signed = storeSigned(op, width);
           break;
         default:
           continue;
@@ -250,6 +249,14 @@ export function returnType(fn: Fn): IrType {
     const term = b.ops[b.ops.length - 1];
     if (term?.opcode === 'ret' && term.operands.length > 0) {
       const v = term.operands[0];
+      // the function's own struct return object, handed back whole (ir/core.ts `RETURNS_OBJECT`)
+      if (term.attrs[RETURNS_OBJECT] === true && v.type.kind === 'ptr') {
+        return v.type.to;
+      }
+      // a struct a call returned by value, handed back whole (frontend/register-struct-call.ts)
+      if (v.type.kind === 'struct') {
+        return v.type;
+      }
       // A NARROW value in the return register does not make the DECLARED return type narrow: the
       // register is a word, and the value's width says how it was COMPUTED, not what the header
       // spelled. The two readings compile alike through agbcc and do NOT through mwcc —