@asmlift/core 0.8.1 → 0.10.0

package/src/raise/paramwidth.ts

@@ -122,11 +122,46 @@
 // pool load in the benchmark's agbcc references (extscale.ts's header has the census). Only the
 // fold's own extensions are judged this way; a plain cast's extension sits at its right shift,
 // where its position says nothing about its `lsl`'s.
-import { type Fn, type Op, type Value, replaceAllUsesWith, successorsOf } from '../ir/core';
+//
+// EVERY READER AN EXTENSION, the `/narrow-param` lift variation (`narrowExtendedParams`). agbcc does
+// not always widen a narrow declared parameter in the prologue: where every use of it is itself an
+// extension it widens at each use instead, and then the object carries no declaration to read.
+// `pokeemerald:LoadMonInfo` declares `s16 partyId` and reads it as `lsl #16` behind a pool load, an
+// `asr #14` for the scaled index and an `lsr #16` for a `u16` argument, so `raw-reader` refuses it
+// (two readers), and with that ablated `not-prologue` (behind the pool load) and `fused-behind-pool`.
+// Declared `s16` and read `a << 2` and `(u16)a`, or declared `s32` and read `(s16)a << 2` and
+// `(u16)a`, the two are one object in isolation and differ only through the register allocation
+// around them (a sibling `u8` parameter's prologue copy), so nothing in the asm decides it and the
+// differ referees. Declaring the parameter at width w changes no reader's value when every reader is
+// an extension to w' ≤ w: ext_w'(trunc_w x) = ext_w'(x). The width is the widest reader's, its
+// signedness the caller's declaration where that spells one and the first reader of that width's
+// otherwise; each reader of exactly that type becomes the parameter itself. Where no prototype
+// declares the width, the candidate loses every score tie (rank.ts `guessesParamWidth`): without
+// a sibling to separate them the two declarations are often one object, and the narrow one costs
+// each prototyped caller the truncation `proto-width` is about.
+//
+// A LOW MASK IS AN EXTENSION READER. mwcc reads a `u16` parameter as `clrlwi rD,rS,16`, which the
+// PPC frontend lifts as `and rS, 0xffff` — as it must, since the same instruction is every other
+// mask too — so a parameter mwcc reads only through it reaches this pass with no `zext` reader at
+// all. `and v, 0xffff` is `zext v {width=16}` value for value, so it counts as one (`extensionOf`),
+// unsigned, and becomes the parameter itself when the parameter is declared `u16`. The two
+// declarations are not one object there: over a `long *a`, mwcc_242_81 compiles
+// `*a = *a & ~b; *a = *a | c;` to `not; and` when `b` and `c` are `unsigned short`, and to `andc` when
+// they are `unsigned long` and the body masks them, at `-O4,p` and at Mario Party 4's unit flags.
+//
+// The rule is about the IR, not the ISA, so it reaches every frontend that lifts a mask as `and`.
+// IDO reads a declared `u16`/`u8` parameter as `sw a0,0(sp); andi a0,a0,0xffff`, and frontend/mips.ts
+// lifts the `andi` as that mask too: `s32 f(u16 a, s32 b){ return a + b; }` scores 3 wide and 0
+// narrowed. The witness gates still decide whether IDO may narrow: a mask applied in place with no
+// home store (`a = a & 0xffff`) is refused by `unhomed-param`. agbcc spells a mask as `and` with a
+// pooled `0xffff` or a `mov #0xff`, so the variation is offered there too, and loses, because agbcc
+// extends a declared narrow parameter with a shift pair: `s32 f(u32 a, s32 b){ return (a & 0xffff)
+// + b; }` scores 0 wide and 6 narrowed.
+import { type Fn, type Op, type Value, defOpMap, replaceAllUsesWith, successorsOf } from '../ir/core';
 import { CAST_WIDTHS, MATERIALIZING_OPS } from '../ir/opcodes';
 import { T } from '../ir/types';
 import { type Gate, firstRejection } from '../l3/gates';
-import { type FnProto, declaredWidth } from '../proto';
+import { type FnProto, declaredSigned, declaredWidth } from '../proto';
 import type { NarrowParamWitness } from '../target';
 
 /** What the gates below judge: one entry parameter and the extension that reads it. */
@@ -141,6 +176,8 @@ export interface NarrowParamCandidate {
   inPrologue: boolean;
   /** reads of the RAW parameter anywhere in the function */
   uses: number;
+  /** reads of it that are not an extension to `width` or narrower */
+  wideReaders: number;
   /** the width the caller's own prototype declares for this parameter, if it declares one */
   declared: number | undefined;
   /** the extension is one raise/extscale.ts re-split from a fused pair whose `shl` the machine ran
@@ -192,14 +229,14 @@ export const PARAM_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
   },
   {
     id: 'not-prologue',
-    why: 'an extension behind body code is where the SOURCE wrote the cast — and on a schedule that interleaves them, the refusing answer',
+    why: 'an extension behind body code may be the source writing the cast, and nothing in the fan referees a narrow signature against the wide one',
     sound: true,
     guardedBy: 'param-width.test.ts: an extension behind a nullary call is body code',
     rejects: (c) => !c.inPrologue,
   },
   {
     id: 'no-declaration-witness',
-    why: 'this compiler spells a narrow declaration and a body cast the same way, so the asm decides nothing',
+    why: 'the compiler spells a narrow declaration and a body cast the same way, so the assembly decides nothing',
     sound: true,
     guardedBy:
       'param-width.test.ts: a compiler whose two spellings are one object refuses the narrowing, measured or not',
@@ -207,14 +244,14 @@ export const PARAM_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
   },
   {
     id: 'unhomed-param',
-    why: 'this compiler homes a narrow DECLARED parameter, so the absent home store proves the declaration was wide',
+    why: 'the compiler stores a narrow declared parameter to the stack on entry, so one it did not store was declared wide',
     sound: true,
     guardedBy: 'param-width.test.ts: a homing compiler refuses the parameter it did not home',
     rejects: (c) => c.witness === 'home-store-and-in-place' && !c.homed,
   },
   {
     id: 'widened-elsewhere',
-    why: 'this compiler widens a narrow DECLARED parameter in the argument register itself, so a widening that lands in a scratch register is not a declaration',
+    why: 'the compiler widens a narrow declared parameter in its own argument register, so a widening into another register is no declaration',
     sound: true,
     guardedBy:
       'param-width.test.ts: \u2026and refuses one it homed but widened SOMEWHERE ELSE \u2014 that is a 64-bit half',
@@ -229,6 +266,96 @@ export const PARAM_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
   },
 ];
 
+const byId = (id: string): Gate<NarrowParamCandidate> => PARAM_WIDTH_GATES.find((g) => g.id === id)!;
+
+/** `/narrow-param`'s table (`narrowExtendedParams`): a second CONSUMER of the default's rules, not an
+ *  ablation of its table, so it is built by id rather than through `ablateHeuristic`. It leaves out
+ *  the three rules that read the prologue's evidence and adds the one that reads the readers.
+ *  `raw-reader` gives way to `wide-reader`, which is sound for the same reason it is: every reader
+ *  is an extension the pass rewrites. `not-prologue` and `fused-behind-pool` read fields this pass
+ *  does not observe (it fills `inPrologue` and `fusedBehindPool` false). Their `sound` is a claim
+ *  about the DEFAULT, whose narrowing reaches every candidate with no wide sibling, so a wrong width
+ *  there is a signature nothing referees and every prototyped caller pays for. A /narrow-param
+ *  candidate keeps the wide default beside it and loses every tie unless a prototype declares the
+ *  width (rank.ts `guessesParamWidth`), so for it the same question is refereed: a rule sound for a
+ *  declaration is a heuristic for a generated candidate (l3/gates.ts, `ORDER_SHAPE_GATES`).
+ *  param-width.test.ts holds the set left out to exactly these three, so a sound rule added to the
+ *  default is either added here or argued out by name. */
+export const PARAM_READER_WIDTH_GATES: readonly Gate<NarrowParamCandidate>[] = [
+  byId('entry-is-join'),
+  byId('param-typed'),
+  byId('cast-width'),
+  {
+    id: 'wide-reader',
+    why: 'a reader that is not an extension to this width or narrower reads bits a narrow declaration drops',
+    sound: true,
+    guardedBy: 'param-width.test.ts: a reader that is no narrow extension keeps the parameter wide',
+    rejects: (c) => c.wideReaders !== 0,
+  },
+  byId('proto-width'),
+  byId('no-declaration-witness'),
+  byId('unhomed-param'),
+  byId('widened-elsewhere'),
+];
+
+const isExtension = (op: Op): boolean => op.opcode === 'sext' || op.opcode === 'zext';
+
+/** The unsigned extension a MASK spells: `and v, 0xff` is `zext v {width=8}` and `and v, 0xffff` is
+ *  `zext v {width=16}`, value for value. */
+const MASK_WIDTHS: ReadonlyMap<number, number> = new Map([
+  [0xff, 8],
+  [0xffff, 16],
+]);
+
+/** What `op` reads `v` as, when it reads it ONLY through an extension: its width and signedness.
+ *  An `and` with a low mask counts, because that is the extension's spelling where the machine
+ *  has no extension of its own to lift: PPC's `clrlwi rD,rS,16` and MIPS's `andi rD,rS,0xffff`
+ *  (each frontend lifts it as the mask, and must, because every other mask is the same instruction).
+ *  Null for any other reader, including an extension or a mask of something besides `v` alone. */
+function extensionOf(op: Op, v: Value, defs: ReadonlyMap<Value, Op>): { width: number; signed: boolean } | null {
+  if (isExtension(op)) {
+    return op.operands.length === 1 && op.operands[0] === v
+      ? { width: op.attrs.width as number, signed: op.opcode === 'sext' }
+      : null;
+  }
+  if (op.opcode !== 'and' || op.operands.length !== 2 || !op.operands.includes(v)) {
+    return null;
+  }
+  const mask = defs.get(op.operands[op.operands[0] === v ? 1 : 0]);
+  const width = mask?.opcode === 'const' ? MASK_WIDTHS.get(mask.attrs.value as number) : undefined;
+  return width === undefined ? null : { width, signed: false };
+}
+
+/** The ops that read `v` and the number of branch arguments that pass it, anywhere in `fn`. */
+function readersOf(fn: Fn, v: Value): { ops: Op[]; edges: number } {
+  const ops: Op[] = [];
+  let edges = 0;
+  for (const b of fn.blocks) {
+    for (const op of b.ops) {
+      if (op.operands.includes(v)) {
+        ops.push(op);
+      }
+      for (const s of op.successors) {
+        edges += s.args.filter((a) => a === v).length;
+      }
+    }
+  }
+  return { ops, edges };
+}
+
+/** How many reads of `v` are not an extension of `v` alone (`extensionOf`) to `width` or narrower. */
+function wideReaderCount(fn: Fn, v: Value, width: number, defs: ReadonlyMap<Value, Op>): number {
+  const { ops, edges } = readersOf(fn, v);
+  return (
+    edges +
+    ops.reduce(
+      (n, op) =>
+        n + ((extensionOf(op, v, defs)?.width ?? Infinity) <= width ? 0 : op.operands.filter((o) => o === v).length),
+      0,
+    )
+  );
+}
+
 /** How many times `v` is read anywhere in `fn` — op operands and branch arguments alike. */
 function useCount(fn: Fn, v: Value): number {
   let n = 0;
@@ -259,6 +386,13 @@ export function narrowEntryParams(
   // function nobody measured (parsed IR, a hand-built fn) is never narrowed on evidence never taken.
   const evidence = fn.paramEvidence;
   const entry = fn.blocks[0];
+  // PER PARAMETER, AND NOT `proto.ts` `declaredCallArgs`, WHICH ASKS A DIFFERENT QUESTION. That
+  // reader abstains for a whole list over one spelling it cannot size, because its caller is
+  // laying out argument registers and one unknown width displaces every later argument. Here every
+  // entry stands alone: `undefined` at position k is the `proto-width` gate ABSTAINING for
+  // parameter k, which is the right answer for a project typedef, and it says nothing about
+  // parameter k+1. A shared reader would make one unreadable spelling drop the narrowing of every
+  // other parameter.
   const declared = Array.isArray(self?.params) ? self.params.map(declaredWidth) : [];
   const entryIsJoin = fn.blocks.some((b) => successorsOf(b).includes(entry));
   const params = new Set(entry.params);
@@ -275,6 +409,7 @@ export function narrowEntryParams(
     }
     prologue.add(op);
   }
+  const defs = defOpMap(fn);
   let narrowed = 0;
   for (const op of [...entry.ops]) {
     if (op.opcode !== 'sext' && op.opcode !== 'zext') {
@@ -291,6 +426,7 @@ export function narrowEntryParams(
       entryIsJoin,
       inPrologue: prologue.has(op),
       uses: useCount(fn, p),
+      wideReaders: wideReaderCount(fn, p, width, defs),
       declared: declared[entry.params.indexOf(p)],
       fusedBehindPool: fusedBehindPool.has(op),
       witness,
@@ -307,3 +443,64 @@ export function narrowEntryParams(
   }
   return narrowed;
 }
+
+/** `/narrow-param`: type each entry parameter every reader of which is an extension of it at the
+ *  widest reader's width, and drop the readers of exactly that type (EVERY READER AN EXTENSION).
+ *  `apply: false` changes nothing and reports what it would narrow. Returns each parameter it
+ *  narrowed. */
+export function narrowExtendedParams(
+  fn: Fn,
+  witness: NarrowParamWitness,
+  self?: FnProto,
+  gates: readonly Gate<NarrowParamCandidate>[] = PARAM_READER_WIDTH_GATES,
+  apply = true,
+): NarrowParamCandidate[] {
+  const evidence = fn.paramEvidence;
+  const entry = fn.blocks[0];
+  const declaredTypes = Array.isArray(self?.params) ? self.params : [];
+  const entryIsJoin = fn.blocks.some((b) => successorsOf(b).includes(entry));
+  const narrowed: NarrowParamCandidate[] = [];
+  const defs = defOpMap(fn);
+  for (const [i, p] of entry.params.entries()) {
+    const extensions = readersOf(fn, p).ops.flatMap((op) => {
+      const ext = extensionOf(op, p, defs);
+      return ext === null ? [] : [{ op, ...ext }];
+    });
+    if (extensions.length === 0) {
+      continue;
+    }
+    const width = Math.max(...extensions.map((e) => e.width));
+    const first = extensions.find((e) => e.width === width)!;
+    const declared = declaredTypes[i];
+    const c: NarrowParamCandidate = {
+      param: p,
+      width,
+      entryIsJoin,
+      inPrologue: false,
+      uses: useCount(fn, p),
+      wideReaders: wideReaderCount(fn, p, width, defs),
+      declared: declared === undefined ? undefined : declaredWidth(declared),
+      fusedBehindPool: false,
+      witness,
+      homed: evidence?.get(p)?.deadHome ?? false,
+      selfRedefined: evidence?.get(p)?.selfRedefined ?? false,
+    };
+    if (firstRejection(gates, c) !== null) {
+      continue;
+    }
+    narrowed.push(c);
+    if (!apply) {
+      continue;
+    }
+    const signed = (declared === undefined ? undefined : declaredSigned(declared)) ?? first.signed;
+    p.type = T.int(width, signed);
+    for (const { op, ...ext } of extensions) {
+      if (ext.width === width && ext.signed === signed) {
+        replaceAllUsesWith(fn, op.results[0], p);
+        const home = fn.blocks.find((b) => b.ops.includes(op))!;
+        home.ops.splice(home.ops.indexOf(op), 1);
+      }
+    }
+  }
+  return narrowed;
+}

package/src/raise/pre-recovery.ts

@@ -12,9 +12,10 @@
 // in the driver.
 import { Block, Fn } from '../ir/core';
 import { simplifyTrivialPhis } from '../ir/simplify';
-import { dce } from '../pattern/engine';
-import type { FnProto } from '../proto';
+import { NOT_FCMP_PATTERNS, applyPattern, dce } from '../pattern/engine';
+import { type FnProto, declaresVoidReturn } from '../proto';
 import type { TargetDescription } from '../target';
+import { readDeclaredMembers } from './aggmembers';
 import { recognizeArrays } from './arrays';
 import { recognizeConsts } from './const';
 import { recognizeDivPow2 } from './divpow2';
@@ -27,16 +28,21 @@ import {
   poolOrderOf,
   restoreUnclaimedScales,
 } from './extscale';
+import { recognizeFloatHelpers } from './floathelpers';
 import { numberPureValues } from './gvn';
 import { recognizeMagicDivision } from './magicdiv';
 import { recognizeMemberArrays } from './memberarrays';
 import { rerootNarrowReads } from './narrow';
 import { type MergeShape, mergeShapes, narrowBlockLocals } from './narrowlocal';
-import { PARAM_WIDTH_GATES, narrowEntryParams } from './paramwidth';
+import { fuseParamPairs, refuseSplitWideParams } from './pairparams';
+import { PARAM_READER_WIDTH_GATES, PARAM_WIDTH_GATES, narrowEntryParams, narrowExtendedParams } from './paramwidth';
 import { type BranchShortCircuitOptions, recognizeBranchShortCircuit, recognizeShortCircuit } from './shortcircuit';
 import { recognizeSoftDiv } from './softdiv';
+import { seedNarrowStoreSigns } from './storesign';
 import { recognizeStructArrays } from './struct-arrays';
-import { recognizeStructs } from './structs';
+import { type DisplacedOptions, foldDisplacedAccesses, recognizeStructs } from './structs';
+import { TRUNC_LOAD_GATES, foldTruncatedLoads } from './truncload';
+import { recognizeWideHelpers, refuseUnmodelledHelpers } from './widehelpers';
 
 /** Per-call options for the pass list — ONE field per pass that takes any, named for the pass, so
  *  a caller reads which recognizer it is steering and a pass that takes none says so by absence.
@@ -45,6 +51,19 @@ import { recognizeStructs } from './structs';
 export interface PreRecoveryOptions {
   /** raise/shortcircuit.ts `recognizeBranchShortCircuit` — the connective-vs-comparison-tree question. */
   shortCircuit?: BranchShortCircuitOptions;
+  /** raise/paramwidth.ts `narrowExtendedParams` — the `/narrow-param` lift. */
+  paramWidth?: ParamWidthOptions;
+  /** raise/structs.ts `foldDisplacedAccesses` — the `/far-cast` lift. */
+  displaced?: DisplacedOptions;
+}
+
+export interface ParamWidthOptions {
+  /** also narrow a parameter every reader of which is an extension (`/narrow-param`) */
+  extendedReaders?: boolean;
+  /** called when a lift without `extendedReaders` has a parameter that would narrow with it */
+  onExtendedReaders?: () => void;
+  /** called when the `extendedReaders` lift narrows a parameter whose width no prototype declares */
+  onUndeclaredWidth?: () => void;
 }
 
 /** CFG facts read off the function ONCE, before the first pass below rewrites it.
@@ -91,8 +110,9 @@ export interface PreRecoveryPass {
 
 /** THE ordered pre-recovery pass list — the single source of truth shared by pipeline / rank / report.
  *  address-numbering → const-materialize → magic-division → pow2-division → soft-division →
- *  scaled-extension → array-legalize → struct-array → member-array → struct-pointer → short-circuit →
- *  branch-short-circuit → narrow-reads → narrow-local → parameter-width → scaled-extension-restore.
+ *  scaled-extension → store-sign → array-legalize → truncated-load → struct-array → member-array →
+ *  struct-pointer → short-circuit → branch-short-circuit → narrow-reads → narrow-local →
+ *  parameter-width → scaled-extension-restore.
  *  See each recognizer's file for the rationale. */
 export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // FIRST: collapsing duplicate address definitions removes block params every later recognizer
@@ -121,7 +141,67 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // ending in `br`. Verified by running it before and after each of them — same result. It sits
   // beside magicdiv so that a reader looking for division recovery finds both together.
   { id: 'divpow2', run: recognizeDivPow2, dce: true },
-  { id: 'softdiv', run: (fn) => recognizeSoftDiv(fn), dce: false, gate: (t) => !t.capabilities.hwDivide },
+  {
+    id: 'softdiv',
+    run: (fn, _self, _opts, target) => recognizeSoftDiv(fn, target),
+    dce: false,
+    gate: (t) => !t.capabilities.hwDivide,
+  },
+  // NOT gated, where `softdiv` is: a float helper computes on floats, which an ISA with no FPU has no
+  // instruction for, and the target's runtime table is what says whether it calls one. A pass apart
+  // from `widehelpers` because it re-types words as floats — a helper's, a parameter a callee
+  // declares `double`, a literal — with its own closed-set refusal, and a single-precision helper
+  // is a float helper that is not wide.
+  // A fold re-runs the float compares' negation idioms (pattern/engine.ts `NOT_FCMP_PATTERNS`),
+  // which ran before there was a float compare to negate.
+  {
+    id: 'floathelpers',
+    run: (fn, _self, _opts, target) => {
+      if (!recognizeFloatHelpers(fn, target)) {
+        return false;
+      }
+      if (NOT_FCMP_PATTERNS.reduce((n, p) => n + applyPattern(fn, p, target), 0) > 0) {
+        dce(fn);
+      }
+      return true;
+    },
+    dce: false,
+  },
+  // NOT gated, where `softdiv` is, and the asymmetry is the argument: a soft DIVISION is
+  // a division the ISA has no instruction for, so a target with a divider never emits one — while
+  // no ISA this repo targets has 64-bit integer arithmetic at all, so a 64-bit helper is software
+  // on every one of them whatever `hwDivide` says.
+  {
+    id: 'widehelpers',
+    run: (fn, _self, _opts, target) => recognizeWideHelpers(fn, target),
+    dce: false,
+  },
+  // AFTER the three helper recognizers, because what it refuses is precisely what they declined: a
+  // call to a name the target's runtime table carries that none of `softdiv`, `floathelpers` and
+  // `widehelpers` folded.
+  // Re-emitting the compiler's own runtime call as source is the one failure that MATCHES, so the
+  // gap has to be written here rather than left to the backend. See raise/widehelpers.ts.
+  //
+  // `dce: false` — an `opaque` carries `effects`, so nothing it replaces becomes dead.
+  {
+    id: 'helperrefuse',
+    run: (fn, _self, _opts, target) => refuseUnmodelledHelpers(fn, target),
+    dce: false,
+  },
+  // AFTER it, because the `concat`s it reads are the ones the frontend built at the call the pass
+  // above has just rewritten — and because a pair the rewrite refused is a pair this must not fuse
+  // into a signature either.
+  // It also refuses a declared 64-bit parameter the lift left in halves, which is the one place the
+  // fusion's answer is final (`refuseSplitWideParams`).
+  {
+    id: 'pairparams',
+    run: (fn, self) => {
+      const fused = fuseParamPairs(fn);
+      refuseSplitWideParams(fn, self);
+      return fused;
+    },
+    dce: false,
+  },
   // AFTER `const`, which folds a shift pair over a constant to the constant it computes, and BEFORE
   // the three array recognizers, whose input this pass produces: `shl(ext(x), k)` is an element
   // scale they legalize and the fused pair is not. `dce: true` — the `shl` a fold leaves readerless.
@@ -133,11 +213,45 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
     dce: true,
     gate: foldsShiftPairCasts,
   },
+  // BEFORE `arrays` and `struct-arrays`, which carry the stamp into the `astore` they make (and
+  // struct-arrays types a field only ever stored from it), and before `structs`, which does the
+  // same for a constant-offset field. `member-arrays` overwrites an astore's `signed` with its
+  // member's (raise/memberarrays.ts says why). `dce: false` — it changes no op.
+  { id: 'storesign', run: (fn, _self, _opts, target) => seedNarrowStoreSigns(fn, target), dce: false },
   { id: 'arrays', run: recognizeArrays, dce: true },
+  // BEFORE the three struct synthesizers, and that is what the pass is FOR: `raise/structs.ts` is
+  // the pass that refuses one offset read at two widths, and it can only see one width per offset
+  // if the fold has already run. Below it the fold is dead — moved after `structs`,
+  // `pokeemerald:AnimTask_FlashHealthboxOnLevelUp_Step:agbcc` declines again on the same throw.
+  //
+  // EVERY POSITION ABOVE `structs` IS FREE, and that is measured rather than argued: two full
+  // `bench sweep --json` runs, this seat against the entry moved to index 0 — above `addrnum`,
+  // `const`, the three division recognizers, `extscale` and `arrays` — agree on all 2,430 records
+  // over 1,215 rows, compared by ROW IDENTITY on `src`/`len`/`diag`/`marks`/`asm`. (The 84
+  // `af:*:ido7.1` records whose `opts` digest churns between any two sweeps are the corpus-wide
+  // defect, which moves whatever anyone does; a LINE diff would report them as disagreement.)
+  //
+  // So the seat is a READING ORDER rather than a constraint: the three struct synthesizers follow
+  // each other here, and a reader looking for what happens to a base's access set finds them
+  // together. The one position that LOOKS like a prerequisite is `addrnum` — the fold groups
+  // accesses by base VALUE identity, so collapsing duplicate address definitions first reads as
+  // one. It is not, and index 0 is the run that shows it.
+  //
+  // `dce: false` — the rewrite retires nothing (the base keeps its use in the load the fold makes).
+  {
+    id: 'truncload',
+    run: (fn, _self, _opts, target) =>
+      foldTruncatedLoads(fn, target.capabilities.endianness === 'little', TRUNC_LOAD_GATES),
+    dce: false,
+  },
   // struct-arrays AFTER arrays (scalar stride==width shapes are claimed first — see the
   // discriminator note in raise/struct-arrays.ts) and BEFORE structs (an element's field
   // accesses must not be re-derived as constant-offset struct-pointer accesses).
-  { id: 'struct-arrays', run: recognizeStructArrays, dce: true },
+  {
+    id: 'struct-arrays',
+    run: (fn, _self, _opts, target) => recognizeStructArrays(fn, target.compilerBehaviors.aggregateBoundary),
+    dce: true,
+  },
   // member-arrays AFTER struct-arrays, and that order IS load-bearing — but the order alone does
   // not deconflict them, because the two passes claim different VALUES: struct-arrays types the
   // materialized `add(P, K)` and member-arrays groups on `P`, so `P->tbl[i].f` reaches both. What
@@ -147,7 +261,28 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // sits there so the three struct synthesizers read in the order of their evidence. `narrowlocal`
   // reads the carrier's extension chain rather than any address, so nothing here orders against it.
   { id: 'member-arrays', run: (fn) => recognizeMemberArrays(fn), dce: true },
-  { id: 'structs', run: recognizeStructs, dce: false },
+  // RIGHT BEFORE `structs`, whose input it widens: an access through a materialised `add(p, K)`
+  // joins `p`'s access set at `off + K`. After `truncload`, `struct-arrays` and `member-arrays`, so
+  // none of them sees a folded access, and a displacement either of the last two typed is refused.
+  // `dce: true` — the `add` chain a fold leaves readerless. `opts.displaced` is `/far-cast`.
+  {
+    id: 'displaced',
+    run: (fn, self, opts, target) =>
+      foldDisplacedAccesses(fn, opts.displaced, target.compilerBehaviors, declaresVoidReturn(self)),
+    dce: true,
+  },
+  {
+    id: 'structs',
+    run: (fn, _self, _opts, target) => recognizeStructs(fn, target.compilerBehaviors.aggregateBoundary),
+    dce: false,
+  },
+  // AFTER `structs`, which synthesizes layouts for bases it can type and never for a frame object
+  // the audit already typed as a declared struct — the only base this reads.
+  {
+    id: 'aggmembers',
+    run: (fn, _self, _opts, target) => readDeclaredMembers(fn, target.capabilities.endianness === 'little'),
+    dce: true,
+  },
   { id: 'shortcircuit', run: recognizeShortCircuit, dce: true },
   // The control-flow sibling, value form FIRST.
   //
@@ -204,16 +339,13 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   // The `target` argument is read by TWO of this pass's gates, and unlike `narrowlocal`'s single
   // conjunct it is not a tuning knob: `narrowParamWitness` says WHICH fact in the object settles a
   // parameter's declared width on this compiler, and a target that names none refuses outright.
+  //
+  // `opts.paramWidth` adds the `/narrow-param` lift after it, over the parameters it left wide
+  // (raise/paramwidth.ts, EVERY READER AN EXTENSION), or asks whether that lift would narrow any.
   {
     id: 'paramwidth',
-    run: (fn, self, _opts, target, lifted) =>
-      narrowEntryParams(
-        fn,
-        target.compilerBehaviors.narrowParamWitness ?? 'none',
-        self,
-        PARAM_WIDTH_GATES,
-        lifted.scales.behindPool,
-      ),
+    run: (fn, self, opts, target, lifted) =>
+      narrowParamWidths(fn, self, opts.paramWidth, target, lifted, PARAM_WIDTH_GATES, PARAM_READER_WIDTH_GATES),
     dce: false,
   },
   // LAST, after every pass that can CLAIM what `extscale` exposed — the two width passes above take
@@ -228,6 +360,31 @@ export const PRE_RECOVERY_PASSES: PreRecoveryPass[] = [
   },
 ];
 
+/** The `paramwidth` entry's body, taking both of its tables, so a census can pass wrapped ones. */
+export function narrowParamWidths(
+  fn: Fn,
+  self: FnProto | undefined,
+  opts: ParamWidthOptions | undefined,
+  target: TargetDescription,
+  lifted: PreRecoveryFacts,
+  prologueGates: typeof PARAM_WIDTH_GATES,
+  readerGates: typeof PARAM_READER_WIDTH_GATES,
+): number {
+  const witness = target.compilerBehaviors.narrowParamWitness ?? 'none';
+  const n = narrowEntryParams(fn, witness, self, prologueGates, lifted.scales.behindPool);
+  if (opts?.extendedReaders) {
+    const lifted = narrowExtendedParams(fn, witness, self, readerGates);
+    if (lifted.some((c) => c.declared === undefined)) {
+      opts.onUndeclaredWidth?.();
+    }
+    return n + lifted.length;
+  }
+  if (opts?.onExtendedReaders && narrowExtendedParams(fn, witness, self, readerGates, false).length > 0) {
+    opts.onExtendedReaders();
+  }
+  return n;
+}
+
 /** Run the pre-recovery passes in order. For each pass whose gate passes and that CHANGES the IR, run
  *  `dce` when the pass declares it, then invoke `afterPass(pass, result)` (the caller's verify/trace
  *  hook), if given.

package/src/raise/pre-structure.ts

@@ -0,0 +1,54 @@
+// asmlift — the CFG folds that run last before structuring, after return-sinking: each rewrites a
+// loop's edges into a shape a loop spelling owns, and each is its own stage in the trace, so a
+// re-spelled function names the fold that moved its CFG.
+//
+// A MUTABLE RECORD, not a sum of four calls, for the reason `PRE_RECOVERY_PASSES` is one: it is the
+// caller-side seam a process outside core swaps to census a fold's gate table (`pnpm bench gates`,
+// apps/benchmark/src/run/gate-census.ts). A static import binding cannot be swapped.
+import { Fn } from '../ir/core';
+import { foldEmptyLatches, foldExitTrampolines } from './latch';
+import { rerollPeeledTests } from './reroll';
+import { unthreadLoopExits } from './unthread';
+
+export interface PreStructureFold {
+  /** stable; the trace stage is `stage:<id>` */
+  readonly id: string;
+  /** the trace stage's title */
+  readonly title: string;
+  /** how many rewrites the fold made; 0 leaves the function as it was */
+  run: (fn: Fn) => number;
+}
+
+/** THE ORDER CARRIES LOAD, and each fold runs ONCE: there is no fixpoint across folds, so a fold
+ *  that exposes another's shape must run before it. The re-roll needs the latch's way out and the
+ *  entry's to reach ONE block, and on `synthetic:break_value:agbcc` they reach it only after the
+ *  trampoline fold has spliced out the empty block the break leaves through; ahead of it, the re-roll
+ *  declines and the loop with it ('unrecovered back-edge'). `reroll.test.ts` pins it by lifting that
+ *  row's pre-fold IR through this record. The latch fold comes first for the same reason in kind:
+ *  the folds after it read the back edge it leaves. The un-threading reads loop exits, which the
+ *  trampoline fold reshapes, and sits ahead of the re-roll so a loop it gives one way out is one
+ *  the re-roll may then read. That place is a precaution: `synthetic:thrbreak:agbcc` and the
+ *  pointer walk unthread.ts names lift the same C wherever the fold sits, so `unthread.test.ts`
+ *  pins its position, not a lift that needs it. */
+export const PRE_STRUCTURE_FOLDS: PreStructureFold[] = [
+  {
+    id: 'latchfold',
+    title: 'Empty-latch folding (a latch that only jumps back becomes its back edge)',
+    run: (fn) => foldEmptyLatches(fn),
+  },
+  {
+    id: 'trampoline',
+    title: 'Exit-trampoline folding (an empty block on a loop exit becomes that edge)',
+    run: (fn) => foldExitTrampolines(fn),
+  },
+  {
+    id: 'unthread',
+    title: 'Loop-exit un-threading (an exit jump-threaded past the re-test after the loop goes back to it)',
+    run: (fn) => unthreadLoopExits(fn),
+  },
+  {
+    id: 'reroll',
+    title: 'Peeled-test re-rolling (a loop test peeled into the entry becomes the first iteration)',
+    run: (fn) => rerollPeeledTests(fn),
+  },
+];