@asmlift/core 0.8.1 → 0.10.0

package/src/backend/cpp.ts

@@ -13,10 +13,14 @@
 // Scope: free functions and non-virtual member functions with scalar/pointer params and named
 // field access. Virtual dispatch, references, and constructors/destructors are deliberately not
 // built ahead of an inhabitant.
-import { Expr, LanguageBackend, SFn } from '../l3/ast';
-import { type CppType, mangle, spellType } from '../mangle';
+import { Expr, LanguageBackend, SFn, walkExprs } from '../l3/ast';
+import { type CppType, declareCpp, mangle, spellType } from '../mangle';
+import type { TargetDescription } from '../target';
 import { LeafHook, cComment, emitCFamily } from './cfamily';
 
+/** How the target's argument slots hold floats (`TargetDescription.fpu`). */
+type FloatSlots = NonNullable<TargetDescription['fpu']>['slots'];
+
 export interface CppClass {
   fields: { name: string; type: CppType }[];
 } // field i at word offset i
@@ -33,25 +37,127 @@ export function cppSymbol(spec: CppFnSpec): string {
   return mangle({ name: spec.method, cls: spec.cls, params: spec.params.map((p) => p.type) });
 }
 
-/** Build a C++ backend for one function, parameterized by its recovered C++ signature. The lifted
- *  SFn params are positional: for a member function SFn.params[0] is `this`, the rest are `params`. */
-export function cppBackend(spec: CppFnSpec): LanguageBackend {
+/** The lifted parameter each EXPLICIT spec parameter is (its SFn name), for a member function after
+ *  `this` (SFn.params[0]). How depends on the target's float slot model (`TargetDescription.fpu`),
+ *  because the lifted order is the ABI sort's (frontend/fpu.ts `fpuArgSlots`):
+ *   - `'separate'` (PowerPC EABI): by register FILE, then by position within it. The files count
+ *     independently and every float argument sorts after the integers, so `float g(float x, int n)`
+ *     lifts as `(s32 a0, float a1)`. A float the body never reads leaves no hole, so the spec may
+ *     have MORE floats than the lift (`int m(int n, float x)` lifts as `(s32 a0)`), never fewer.
+ *   - `'leading'` (MIPS o32), and a target with no float file: by POSITION. The lifted order is the
+ *     slot order, which is the source order, and an unread leading float is minted as an integer
+ *     hole — so binding by file would hand `int k(float x, int n)`'s `n` the hole `a0`.
+ *  NULL when the two contradict: a lifted float the spec has no float for, or has one of the other
+ *  precision for. No binding of the rest is then trustworthy. The precision is part of the
+ *  contradiction because the body is spelled over the spec's types: a lifted `double` bound to a
+ *  `float a, float b` prints `a + b`, which is a single-precision add where the machine did a
+ *  double one.
+ *
+ *  THE PRECISION IS AN OPERATION'S, so only a parameter the body reads has one to contradict. A
+ *  frontend types every float register of a function at the function's one precision
+ *  (frontend/fpu.ts `fpPrecision`), and that includes an argument slot the body never reads, which
+ *  it mints only to hold the later arguments' places: `float f1(double a, float b){ return b + b; }`
+ *  is `fadds f1,f2,f2`, and its `a` is a single-precision hole that nothing states the width of. So
+ *  is a lifted float of no stated precision (`T.fUnstated`) anywhere. Either binds a spec float of
+ *  both widths, and the spec's is the one to print.
+ *
+ *  EXCEPT WHERE THE HOLE'S WIDTH PLACED THE PARAMETERS AFTER IT. Under `'leading'` a float takes one
+ *  integer slot for a single and two for a double, counted at the function's precision
+ *  (`fpuArgSlots`), so an unread hole of the other width than the spec's moves every integer after
+ *  it: `float m2(double a, float b, int *p, int *q){ *p = 0; return b + b; }` (gcc2.7.2kmc) is
+ *  `sw zero,0(a3)` with `a` laid out as ONE slot, and binding by position would name `a3` `q`. So
+ *  there an unread float followed by an integer parameter keeps its precision.
+ *
+ *  A spec `double` bound to a lifted NON-float is the same misplacement. A double takes two integer
+ *  slots under o32 and on a target with no float file (agbcc passes one in r0:r1), so a body that
+ *  reads no FPU register mints it as two 32-bit holes: `void n1(double d, int *p, int *q){ *p = 0; }`
+ *  (gcc2.7.2kmc) is `sw zero,0(a2)` and lifts as `(s32 a0, s32 a1, s32 *a2)`, where binding by
+ *  position would name `a2` `q`. Only a lifted 64-bit parameter holds both slots, so a spec double
+ *  over a narrower one refuses when a lifted parameter follows it, and when the body reads it — a
+ *  read half would print as the whole double (agbcc `int u1(int a, double d)` returning the word
+ *  `((int *)&d)[0]` is `add r0,r1,#0`). A lifted parameter after it is not a refusal when the double
+ *  is the spec's last and the body reads none of the slots from it on, which is IDO homing a
+ *  trailing double it never uses (`int o1(int a, double d){ return a; }` spills `a2`/`a3`): nothing
+ *  then binds past it. */
+export function bindSpecParams(
+  spec: Pick<CppFnSpec, 'cls' | 'params'>,
+  lifted: Pick<SFn, 'params' | 'body'>,
+  floatSlots: FloatSlots | undefined,
+): (string | undefined)[] | null {
+  const floatBits = (t: CppType) => (t.ptr !== 0 ? null : t.base === 'float' ? 32 : t.base === 'double' ? 64 : null);
+  const isFloat = (t: CppType) => floatBits(t) !== null;
+  const read = new Set<string>();
+  for (const e of walkExprs(lifted.body)) {
+    if (e.k === 'var') {
+      read.add(e.name);
+    }
+  }
+  const clashes = (p: SFn['params'][number], t: CppType | undefined, laysOut: boolean) =>
+    p.type.kind === 'float' &&
+    (t === undefined ||
+      floatBits(t) === null ||
+      (p.type.width !== null && (laysOut || read.has(p.name)) && floatBits(t) !== p.type.width));
+  const explicit = lifted.params.slice(spec.cls ? 1 : 0);
+  const lastAndUnread = (i: number) =>
+    i === spec.params.length - 1 && explicit.slice(i).every((p) => !read.has(p.name));
+  if (floatSlots !== 'separate') {
+    const clash = explicit.some((p, i) => {
+      const t = spec.params[i]?.type;
+      const later = explicit.slice(i + 1);
+      const oneSlot = p.type.kind !== 'float' && !('width' in p.type && p.type.width === 64);
+      return (
+        clashes(
+          p,
+          t,
+          later.some((q) => q.type.kind !== 'float'),
+        ) ||
+        (t !== undefined &&
+          floatBits(t) === 64 &&
+          oneSlot &&
+          (later.length > 0 || read.has(p.name)) &&
+          !lastAndUnread(i))
+      );
+    });
+    return clash ? null : spec.params.map((_, i) => explicit[i]?.name);
+  }
+  const floats = explicit.filter((p) => p.type.kind === 'float');
+  const others = explicit.filter((p) => p.type.kind !== 'float');
+  const specFloats = spec.params.filter((p) => isFloat(p.type));
+  if (specFloats.length < floats.length || floats.some((p, i) => clashes(p, specFloats[i].type, false))) {
+    return null;
+  }
+  let f = 0;
+  let o = 0;
+  return spec.params.map((p) => (isFloat(p.type) ? floats[f++] : others[o++])?.name);
+}
+
+/** Build a C++ backend for one function, parameterized by its recovered C++ signature and the float
+ *  slot model of the target it was lifted for. For a member function SFn.params[0] is `this`; the
+ *  rest bind to `params` by `bindSpecParams`. */
+export function cppBackend(spec: CppFnSpec, floatSlots: FloatSlots | undefined): LanguageBackend {
   return {
     id: 'cpp',
     spellsSwitchFallthrough: true,
+    spellsSwitchBreak: true,
     emit(fn: SFn): string {
       // Map each lifted param var → its C++ meaning: `this` (bare member access) or a named param
       // (a pointer-to-class param uses `->`). A pointer-to-known-class param is a member receiver.
       const thisVar = spec.cls ? fn.params[0]?.name : undefined;
-      const explicitStart = spec.cls ? 1 : 0;
       const rename = new Map<string, string>(); // lifted var → C++ name
       const recv = new Map<string, { cls: string; via: 'this' | string }>(); // var → member receiver
       if (thisVar) {
         rename.set(thisVar, 'this');
         recv.set(thisVar, { cls: spec.cls!, via: 'this' });
       }
+      const bound = bindSpecParams(spec, fn, floatSlots);
+      if (!bound) {
+        throw new Error(
+          `cpp backend: the spec's floating-point parameters do not match the lifted function's — ` +
+            `supply a signature whose float and integer parameters are the ones it takes`,
+        );
+      }
       spec.params.forEach((p, i) => {
-        const v = fn.params[explicitStart + i]?.name;
+        const v = bound[i];
         if (!v) {
           return;
         }
@@ -115,10 +221,10 @@ export function cppBackend(spec: CppFnSpec): LanguageBackend {
         return null;
       };
 
-      const paramList = spec.params.map((p) => `${spellType(p.type)} ${p.name}`).join(', ');
+      const paramList = spec.params.map((p) => declareCpp(p.type, p.name)).join(', ');
       const decls = classDecls(spec, paramList);
       const signature = `${spellType(spec.retType)} ${spec.cls ? spec.cls + '::' : ''}${spec.method}(${paramList})`;
-      return (decls ? decls + '\n' : '') + emitCFamily(signature, fn, leaf);
+      return (decls ? decls + '\n' : '') + emitCFamily(signature, fn, leaf, false);
     },
     comment: cComment, // C++ shares C's block-comment spelling
   };
@@ -140,7 +246,7 @@ function typeWidth(t: CppType): number {
 function classDecls(spec: CppFnSpec, paramList: string): string {
   const out: string[] = [];
   for (const [cname, cdef] of Object.entries(spec.classes ?? {})) {
-    const fields = cdef.fields.map((f) => `${spellType(f.type)} ${f.name};`).join(' ');
+    const fields = cdef.fields.map((f) => `${declareCpp(f.type, f.name)};`).join(' ');
     const method = cname === spec.cls ? ` ${spellType(spec.retType)} ${spec.method}(${paramList});` : '';
     out.push(`struct ${cname} { ${fields}${method} };`);
   }

package/src/backend/pascal.ts

@@ -52,6 +52,13 @@ function pasType(t: IrType): string {
     return '^' + pasType(t.to);
   }
   if (t.kind === 'int') {
+    // `Integer`/`Cardinal` are the machine word, and a NARROWER parameter or return fits in one,
+    // which is what this backend spells it as (a narrow local is refused where it is declared). A WIDER one does not: upas has no verified 64-bit type,
+    // and spelling a 64-bit value `Integer` would silently drop its high half. Loud, like every
+    // other unspellable construct in this backend.
+    if (t.width > 32) {
+      throw new Error(`pascal backend: no spelling for a ${t.width}-bit integer`);
+    }
     return t.signed ? 'Integer' : 'Cardinal';
   }
   // `unknown` reaching a backend means recovery's totality contract already failed upstream —
@@ -81,7 +88,17 @@ function makePrinter(vt: VarTypes) {
         throw new Error(`pascal backend: address-of global '${e.name}' has no IDO Pascal spelling yet`);
       case 'const':
         return String(e.value);
+      case 'fconst':
+        throw new Error('pascal backend: a double literal has no IDO Pascal spelling yet');
       case 'call':
+        if (typeof e.fn !== 'string') {
+          throw new Error('pascal backend: a call through a pointer has no IDO Pascal spelling yet');
+        }
+        if (e.sret !== undefined) {
+          throw new Error(
+            `pascal backend: the call to '${e.fn}' returns a struct through memory, which has no spelling here`,
+          );
+        }
         return `${e.fn}(${e.args.map(pe).join(', ')})`;
       case 'index': {
         // The width-carrying access node (l3/ast.ts): each backend legalizes its own derefs. The
@@ -112,14 +129,21 @@ function makePrinter(vt: VarTypes) {
       case 'field':
         throw new Error(`pascal backend: struct field access '${e.name}' has no IDO Pascal spelling yet`);
       case 'un':
+        if (e.op === 'f-') {
+          throw new Error(`pascal backend: a float negation has no faithful IDO Pascal spelling`);
+        }
         return e.op === '~' ? `bitnot(${pe(e.e)})` : `(${e.op === '!' ? 'not ' : e.op}${pe(e.e)})`;
+      // IDO Pascal has no increment operator and no expression with a side effect to put one in,
+      // so the update has no place to go — decline LOUD rather than drop it, like the casts below.
+      case 'postincr':
+        throw new Error(`pascal backend: post-increment of '${e.name}' has no IDO Pascal spelling yet`);
       // Casts have no faithful IDO-Pascal spelling yet — fail LOUD rather than emit silently-wrong
       // source. Tree-level producers reaching here: the width-narrowing idiom casts (agbcc-gated,
       // so never on this path today), structure.ts's STRUCT-pointer casts (unreachable too — the
-      // `field` case above throws first), intify's `(s32)ptr` legalization (any target), and the
-      // byte-pointer walk of a pointer offset by a runtime value (any target, and reachable with no
-      // `field` in the tree — it declines two functions that used to emit here, whose Pascal was
-      // silently walking ELEMENTS where the asm walked bytes).
+      // `field` case above throws first), the arithmetic table's `(s32)ptr` legalization (any
+      // target), and the byte-pointer walk of a pointer offset by a runtime value (any target,
+      // and reachable with no `field` in the tree, where a Pascal spelling would walk ELEMENTS
+      // where the asm walked bytes).
       // Scalar deref casts never appear in the tree — the index case above owns that judgment.
       case 'cast':
         throw new Error(`pascal backend: cast has no IDO Pascal spelling yet`);
@@ -178,13 +202,20 @@ function makePrinter(vt: VarTypes) {
         return s.value ? [`${indent}${fnName} := ${pe(s.value)};`] : [];
       case 'if': {
         const cond = pe(s.cond);
-        const block = (stmts: Stmt[], ind: string) =>
-          stmts.length === 1
+        // A branch goes bare only where it is ONE Pascal statement that cannot capture what follows.
+        // A `for` prints as two (its init, then a `while`), so it always takes begin/end. And with an
+        // `else` to come, a then-branch that is itself an `if` or a loop (whose body may end in one)
+        // would take that `else` as its own innermost `if`'s — Pascal's dangling else, which C's
+        // braces had ruled out: `if (a) { if (b) X; } else Y;` printed bare runs Y when b fails.
+        const bare = (stmts: Stmt[], open: boolean) =>
+          stmts.length === 1 && stmts[0].k !== 'for' && !(open && (stmts[0].k === 'if' || isLoop(stmts[0])));
+        const block = (stmts: Stmt[], ind: string, open = false) =>
+          bare(stmts, open)
             ? ps(fnName, stmts[0], ind, tail)
             : [`${ind}begin`, ...list(stmts, ind + '  ', tail), `${ind}end`];
         // Pascal: no `;` before `else`; the branch statements already carry their own.
         const out = [`${indent}if ${cond} then`];
-        out.push(...block(s.then, indent + '  '));
+        out.push(...block(s.then, indent + '  ', s.else.length > 0));
         if (s.else.length) {
           out.push(`${indent}else`);
           out.push(...block(s.else, indent + '  '));
@@ -194,7 +225,7 @@ function makePrinter(vt: VarTypes) {
       case 'while': {
         // loop bodies are NEVER tail position — control returns to the test
         const body =
-          s.body.length === 1
+          s.body.length === 1 && s.body[0].k !== 'for'
             ? ps(fnName, s.body[0], indent + '  ')
             : [`${indent}begin`, ...s.body.flatMap((x) => ps(fnName, x, indent + '  ')), `${indent}end`];
         return [`${indent}while ${pe(s.cond)} do`, ...body];
@@ -229,7 +260,7 @@ function makePrinter(vt: VarTypes) {
         // so a `fallsThrough` arm has no faithful spelling → loud-fail (mirrors the `field`/`cast`/`%`
         // throws above). Each arm's body is a single statement or a begin/end block.
         const arm = (body: Stmt[], ind: string) =>
-          body.length === 1
+          body.length === 1 && body[0].k !== 'for'
             ? ps(fnName, body[0], ind + '  ')
             : [`${ind}  begin`, ...body.flatMap((x) => ps(fnName, x, ind + '  ')), `${ind}  end`];
         // Pascal has no early `return`: a `return v` inside a case renders `fnName := v` and then FALLS
@@ -255,8 +286,11 @@ function makePrinter(vt: VarTypes) {
           }
           out.push(`${indent}  ${c.values.join(', ')}:`, ...arm(c.body, indent + '  '));
         }
-        if (s.default) {
-          out.push(`${indent}  otherwise`);
+        // A C `switch` with no `default` runs on past a value it does not list; IDO Pascal's
+        // `case` raises a run-time `caseerror` there instead. So the clause is always printed, an
+        // empty one when the source had no default.
+        out.push(`${indent}  otherwise`);
+        if (s.default?.length) {
           out.push(...arm(s.default, indent + '  '));
         }
         out.push(`${indent}end;`);
@@ -274,10 +308,26 @@ export const pascalBackend: LanguageBackend = {
   // spells as plain if-nesting, which Pascal prints, so the choice is between a decompiled
   // function and a stub.
   spellsSwitchFallthrough: false,
+  // Nor has Pascal a `break`, so recovery must not mint a switch's mid-arm one either: a closed arm
+  // that leaves early spells as nested `if`s that copy the arm's tail into each path.
+  spellsSwitchBreak: false,
   emit(fn0: SFn): string {
     // The declaration list is put into the target's own frame order HERE, as the C family does it
     // in its shared assembler — owned by `emit`, never by a `.emit(` call site (l3/slotorder.ts).
     const fn = orderSlotLocals(fn0);
+    // A function-scope static is a definition the MIPS frontends never recover, and IDO Pascal's
+    // spelling of one (a typed constant, or a unit-level var) has not been measured — loud. So is
+    // the compiler's anonymous constant a local's initializer is copied from (l3/constinit.ts).
+    if (fn.statics?.length) {
+      throw new Error(
+        `pascal backend: the function-scope static '${fn.statics[0].name}' has no IDO Pascal spelling yet`,
+      );
+    }
+    if (fn.images?.length) {
+      throw new Error(
+        `pascal backend: the compiler's anonymous constant '${fn.images[0].name}' a local is initialized from has no IDO Pascal spelling yet`,
+      );
+    }
     // Same env discipline as the C family (cfamily.ts cFamilyBody): the printer judges derefs
     // against the exact declarations it emits.
     const ps = makePrinter(declaredTypes(fn));
@@ -290,6 +340,14 @@ export const pascalBackend: LanguageBackend = {
         : `function ${fn.name}(${params}): ${pasType(fn.retType)};`,
     ];
     if (fn.locals.length) {
+      // A narrow LOCAL truncates every value assigned to it, and `Integer`/`Cardinal` would not: the
+      // width is the declaration's whole meaning, so it has no spelling here. A narrow parameter
+      // arrives already narrowed and keeps the word spelling.
+      for (const l of fn.locals) {
+        if (l.type.kind === 'int' && l.type.width < 32) {
+          throw new Error(`pascal backend: no spelling for a narrow local (${l.type.width} bits)`);
+        }
+      }
       lines.push('var', ...fn.locals.map((l) => `  ${l.name}: ${pasType(l.type)};`));
     }
     lines.push('begin');

package/src/cdecl/constant.ts

@@ -0,0 +1,59 @@
+// asmlift — the INTEGER CONSTANTS a declaration writes: a literal's value, and the value of a constant
+// expression of literals, as an array's extent or a bit-field's width is written.
+import type { Tokens } from './lex';
+import type { Range } from './parse';
+
+/** An integer literal's value and its suffix (`u`, `l`, `ll` in any case and order), read as C reads
+ *  it: `0x` hexadecimal, a leading `0` octal. Undefined for any other text: `08`, `1.5`, `SIZE`. */
+export function integerLiteral(text: string): { value: bigint; suffix: string } | undefined {
+  const m = /^(?:0x([0-9a-f]+)|0([0-7]*)|([1-9]\d*))([ul]*)$/i.exec(text);
+  if (m === null) {
+    return undefined;
+  }
+  const [, hex, octal, decimal, suffix] = m;
+  const value =
+    hex !== undefined ? BigInt(`0x${hex}`) : octal !== undefined ? BigInt(`0o${octal || '0'}`) : BigInt(decimal);
+  return { value, suffix };
+}
+
+/** The value of the integer constant expression in `r`, of literals, `+ - * /` and parentheses — the
+ *  `u8 pad3[0x4 - 0x3]` a decomp header sizes its padding with — or undefined for anything else. */
+export function constantValue(tokens: Tokens, r: Range): number | undefined {
+  let at = r.from;
+  const next = (): string | undefined => (at < r.to ? tokens.text(at++) : undefined);
+  const primary = (): number | undefined => {
+    const k = at;
+    const t = next();
+    if (t === '(') {
+      const v = sum();
+      return next() === ')' ? v : undefined;
+    }
+    if (t === '-') {
+      const v = primary();
+      return v === undefined ? undefined : -v;
+    }
+    const literal = t !== undefined && tokens.kind(k) === 'number' ? integerLiteral(t) : undefined;
+    return literal === undefined ? undefined : Number(literal.value);
+  };
+  const product = (): number | undefined => {
+    let v = primary();
+    while (v !== undefined && at < r.to && (tokens.is(at, '*') || tokens.is(at, '/'))) {
+      const op = next();
+      const right = primary();
+      v =
+        right === undefined || (op === '/' && right === 0) ? undefined : op === '*' ? v * right : Math.trunc(v / right);
+    }
+    return v;
+  };
+  const sum = (): number | undefined => {
+    let v = product();
+    while (v !== undefined && at < r.to && (tokens.is(at, '+') || tokens.is(at, '-'))) {
+      const op = next();
+      const right = product();
+      v = right === undefined ? undefined : op === '+' ? v + right : v - right;
+    }
+    return v;
+  };
+  const v = sum();
+  return at === r.to ? v : undefined;
+}

package/src/cdecl/lex.ts

@@ -0,0 +1,209 @@
+// asmlift — the TOKENS of a C or C++ declaration context: one pass that decides what is code.
+//
+// Whitespace, comments and preprocessor lines (continuations included) are not tokens: a context is
+// preprocessed, so a `#` line left in it is a line marker or a pragma, and a raw header's
+// conditionals are read as if every branch were taken. A string or character literal is one token,
+// and ends at its newline when nothing closes it. `::` and `...` are one token each; every other
+// punctuator is one character, so `>>` closing two template argument lists is two `>`.
+//
+// Brackets are matched once, here: a `}` closes everything opened since its `{`, and a `)` or `]`
+// that does not close the innermost open bracket matches nothing. So an unbalanced `(` inside a
+// body cannot carry the rest of the context with it.
+//
+// Storage is one typed array per field, indexed by token: a context is up to a few megabytes, and an
+// object per token is what would make reading it slow.
+
+export type TokenKind = 'identifier' | 'number' | 'string' | 'punct' | 'end';
+
+export interface Tokens {
+  readonly src: string;
+  readonly count: number;
+  /** `end` past the last token */
+  kind(i: number): TokenKind;
+  text(i: number): string;
+  is(i: number, s: string): boolean;
+  /** a punctuator's first character code, or -1 for any other token */
+  char(i: number): number;
+  /** the matching close of an open `{ ( [`, the open of a close, or -1 */
+  match(i: number): number;
+  /** the source offset of the token's first character */
+  start(i: number): number;
+  /** the source offset just past the token */
+  end(i: number): number;
+}
+
+const KINDS: readonly TokenKind[] = ['end', 'identifier', 'number', 'string', 'punct'];
+const IDENTIFIER = 1;
+const NUMBER = 2;
+const STRING = 3;
+const PUNCT = 4;
+
+const isIdentifierStart = (c: number): boolean => (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || c === 95 || c === 36;
+const isDigit = (c: number): boolean => c >= 48 && c <= 57;
+/** whether the newline at `i` ends a line that a backslash continues */
+const continued = (src: string, i: number): boolean =>
+  src.charCodeAt(i - 1) === 92 || (src.charCodeAt(i - 1) === 13 && src.charCodeAt(i - 2) === 92);
+
+class TokenArrays implements Tokens {
+  readonly src: string;
+  readonly count: number;
+  readonly #kind: Uint8Array;
+  readonly #start: Int32Array;
+  readonly #end: Int32Array;
+  readonly #match: Int32Array;
+
+  constructor(src: string, count: number, kind: Uint8Array, start: Int32Array, end: Int32Array, match: Int32Array) {
+    this.src = src;
+    this.count = count;
+    this.#kind = kind;
+    this.#start = start;
+    this.#end = end;
+    this.#match = match;
+  }
+
+  kind(i: number): TokenKind {
+    return i < this.count ? KINDS[this.#kind[i]] : 'end';
+  }
+
+  text(i: number): string {
+    return this.src.slice(this.#start[i], this.#end[i]);
+  }
+
+  is(i: number, s: string): boolean {
+    return i < this.count && this.#end[i] - this.#start[i] === s.length && this.src.startsWith(s, this.#start[i]);
+  }
+
+  char(i: number): number {
+    return i < this.count && this.#kind[i] === PUNCT ? this.src.charCodeAt(this.#start[i]) : -1;
+  }
+
+  match(i: number): number {
+    return i < this.count ? this.#match[i] : -1;
+  }
+
+  start(i: number): number {
+    return i < this.count ? this.#start[i] : this.src.length;
+  }
+
+  end(i: number): number {
+    return i < this.count ? this.#end[i] : this.src.length;
+  }
+}
+
+export function lex(src: string): Tokens {
+  let capacity = Math.max(16, src.length >> 2);
+  let kind = new Uint8Array(capacity);
+  let start = new Int32Array(capacity);
+  let end = new Int32Array(capacity);
+  let count = 0;
+  const push = (k: number, from: number, to: number): void => {
+    if (count === capacity) {
+      capacity *= 2;
+      const k2 = new Uint8Array(capacity);
+      k2.set(kind);
+      kind = k2;
+      const s2 = new Int32Array(capacity);
+      s2.set(start);
+      start = s2;
+      const e2 = new Int32Array(capacity);
+      e2.set(end);
+      end = e2;
+    }
+    kind[count] = k;
+    start[count] = from;
+    end[count] = to;
+    count++;
+  };
+  const length = src.length;
+  // only whitespace and comments since the last newline: a `#` here starts a directive
+  let lineStart = true;
+  let i = 0;
+  while (i < length) {
+    const c = src.charCodeAt(i);
+    if (c === 10) {
+      lineStart = true;
+      i++;
+    } else if (c === 32 || c === 9 || c === 13 || c === 12 || c === 11) {
+      i++;
+    } else if (c === 47 && src.charCodeAt(i + 1) === 47) {
+      const newline = src.indexOf('\n', i);
+      i = newline < 0 ? length : newline;
+    } else if (c === 47 && src.charCodeAt(i + 1) === 42) {
+      const close = src.indexOf('*/', i + 2);
+      i = close < 0 ? length : close + 2;
+    } else if (c === 35 && lineStart) {
+      while (i < length && !(src.charCodeAt(i) === 10 && !continued(src, i))) {
+        i++;
+      }
+    } else {
+      lineStart = false;
+      let j = i + 1;
+      if (isIdentifierStart(c)) {
+        while (j < length && (isIdentifierStart(src.charCodeAt(j)) || isDigit(src.charCodeAt(j)))) {
+          j++;
+        }
+        push(IDENTIFIER, i, j);
+      } else if (isDigit(c)) {
+        // a preprocessing number: `0x1F`, `1.5f`, `1e-3`
+        for (; j < length; j++) {
+          const d = src.charCodeAt(j);
+          const sign = (d === 43 || d === 45) && /[eEpP]/.test(src[j - 1]);
+          if (!(isIdentifierStart(d) || isDigit(d) || d === 46 || sign)) {
+            break;
+          }
+        }
+        push(NUMBER, i, j);
+      } else if (c === 34 || c === 39) {
+        while (j < length && src.charCodeAt(j) !== c && src.charCodeAt(j) !== 10) {
+          j += src.charCodeAt(j) === 92 ? 2 : 1;
+        }
+        if (j < length && src.charCodeAt(j) === c) {
+          j++;
+        }
+        push(STRING, i, j);
+      } else if (c === 58 && src.charCodeAt(j) === 58) {
+        j++;
+        push(PUNCT, i, j);
+      } else if (c === 46 && src.charCodeAt(j) === 46 && src.charCodeAt(j + 1) === 46) {
+        j += 2;
+        push(PUNCT, i, j);
+      } else {
+        push(PUNCT, i, j);
+      }
+      i = j;
+    }
+  }
+  return new TokenArrays(src, count, kind, start, end, matchBrackets(src, count, kind, start));
+}
+
+function matchBrackets(src: string, count: number, kind: Uint8Array, start: Int32Array): Int32Array {
+  const match = new Int32Array(count).fill(-1);
+  const open: number[] = [];
+  for (let t = 0; t < count; t++) {
+    if (kind[t] !== PUNCT) {
+      continue;
+    }
+    const c = src.charCodeAt(start[t]);
+    if (c === 123 || c === 40 || c === 91) {
+      open.push(t);
+    } else if (c === 125) {
+      let k = open.length - 1;
+      while (k >= 0 && src.charCodeAt(start[open[k]]) !== 123) {
+        k--;
+      }
+      if (k >= 0) {
+        match[open[k]] = t;
+        match[t] = open[k];
+        open.length = k;
+      }
+    } else if (c === 41 || c === 93) {
+      const top = open[open.length - 1];
+      if (top !== undefined && src.charCodeAt(start[top]) === (c === 41 ? 40 : 91)) {
+        match[top] = t;
+        match[t] = top;
+        open.pop();
+      }
+    }
+  }
+  return match;
+}