@scriptc/compiler 0.4.6 → 0.4.8

package/dist/frontend/lowering/unit-journal.js

@@ -4,8 +4,9 @@
  * unit's lowering reached: the unit's IR, the entries it added to the
  * tables that collect lowering results (Lowerer.journalTables), and the
  * calls it made into reachability, generic demand and statement counting
- * (JournalEvent). Records hold only strings, numbers and names, so they can
- * be written to a file and read by another process or another build.
+ * (JournalEvent). Records hold only the function's bytes (or its JSON),
+ * strings, numbers and names, so they can be written to a file and read by
+ * another process or another build.
  *
  * A unit has no record when its lowering changed state that no record
  * carries: a table that is not appendable grew, a field changed, an entry
@@ -23,16 +24,42 @@
 import { appendFileSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import { InternalCompilerError } from "../../errors.js";
+import { decodeFunctionBinary, encodeFunctionStrings, functionDigest } from "../../ir/codec.js";
 import { deserializeFunction, encodeFunction } from "../../ir/serialize.js";
-/** A function as records hold it, or null when it does not encode. Every
- * function a record holds goes through this pair. */
+import { invalidateTrackedPath } from "../input-tracker.js";
+/** A function as table entries hold it, or null when it does not encode.
+ * Every function an entry holds goes through this pair. */
 export function encodeUnitFunction(fn) {
     return encodeFunction(fn) ?? null;
 }
-/** A function from its record encoding (encodeUnitFunction). */
+/** A function from its entry encoding (encodeUnitFunction). */
 export function decodeUnitFunction(text) {
     return deserializeFunction(text);
 }
+/** `fn` as a record holds it, or null when it does not encode. A binary
+ * encoding comes with its strings and the files its fields named `file`
+ * hold (encodeFunctionStrings). */
+export function encodeRecordFunction(fn) {
+    const binary = encodeFunctionStrings(fn);
+    if (binary !== null)
+        return { fn: binary.bytes, strings: binary.strings, files: binary.files };
+    const text = encodeUnitFunction(fn);
+    return text === null ? null : { fn: text, strings: null, files: null };
+}
+/** A function from its record encoding (encodeRecordFunction). */
+export function decodeRecordFunction(fn) {
+    return typeof fn === "string"
+        ? decodeUnitFunction(fn)
+        : decodeFunctionBinary(Buffer.from(fn.buffer, fn.byteOffset, fn.byteLength));
+}
+/** `record` as a JSON line, with a binary function in base64, marked when
+ * it was `applied` from an earlier build's store. */
+export function recordLine(record, applied = false) {
+    const fn = record.fn === null || typeof record.fn === "string"
+        ? record.fn
+        : Buffer.from(record.fn.buffer, record.fn.byteOffset, record.fn.byteLength).toString("base64");
+    return JSON.stringify(applied ? { ...record, fn, applied: true } : { ...record, fn });
+}
 /** A digest of `text`: its length and two 32-bit FNV-1a hashes. */
 export function contentDigest(text) {
     let a = 0x811c9dc5;
@@ -44,10 +71,20 @@ export function contentDigest(text) {
     }
     return `${text.length}:${(a >>> 0).toString(16)}${(b >>> 0).toString(16)}`;
 }
-/** The names quoted in JSON `texts`, unescaped only as far as their
- * delimiting quotes: the strings that start with `%` (helper, lifted and
- * IR data names) and union ids (`u` and 12 hex digits), which the tables
- * a unit reads and changes are keyed by. */
+/** Adds to `out` the names among `strings`: the strings that start with `%`
+ * (helper, lifted and IR data names) and union ids (`u` and 12 hex digits),
+ * which the tables a unit reads and changes are keyed by. */
+export function namedStrings(strings, out) {
+    for (const text of strings) {
+        const first = text.charCodeAt(0);
+        if (first === 37 /* % */)
+            out.add(text);
+        else if (first === 117 /* u */ && text.length === 13 && /^u[0-9a-f]{12}$/.test(text))
+            out.add(text);
+    }
+}
+/** The names (namedStrings) quoted in JSON `texts`, unescaped only as far
+ * as their delimiting quotes. */
 export function quotedStrings(texts) {
     const out = new Set();
     const hex = (char) => (char >= 48 && char <= 57) || (char >= 97 && char <= 102);
@@ -106,6 +143,12 @@ export function flagScalar(name, get, set) {
 /** The events of the unit being recorded (Lowerer.unitEvents). */
 export class UnitEvents {
     events = [];
+    /** Entries the unit changes in place, which its IR cannot show: a table
+     * only grows by addition, so a write to an entry another unit added is not
+     * in the unit's function. recordUnit merges these into the names it reads
+     * from the unit's IR, so the unit is counted as changing what it wrote and
+     * the held content follows (HeldTables). */
+    touched = new Set();
     /** Calls into reachability nest (an edge can fire virtual edges). Only
      * the outermost call is recorded, because making it again makes the
      * nested ones. */
@@ -114,6 +157,10 @@ export class UnitEvents {
         if (this.depth === 0)
             this.events.push({ kind, name, detail, index });
     }
+    /** Notes that the unit writes `name`, an entry it did not add. */
+    touch(name) {
+        this.touched.add(name);
+    }
 }
 /** Whether reachable emit records its units (SCRIPTC_LOWER_JOURNAL=serial,
  * or a SCRIPTC_LOWER_JOURNAL_DUMP directory). */
@@ -133,10 +180,25 @@ export function writeRecords(dir, lines) {
     if (lines.length === 0)
         return;
     mkdirSync(dir, { recursive: true });
-    appendFileSync(join(dir, "records.jsonl"), `${lines.join("\n")}\n`);
+    const path = join(dir, "records.jsonl");
+    appendFileSync(path, `${lines.join("\n")}\n`);
+    invalidateTrackedPath(path);
 }
 export function waveStats() {
-    return { waves: 0, units: 0, recorded: 0, unrecorded: 0, reasons: {} };
+    return {
+        waves: 0,
+        units: 0,
+        recorded: 0,
+        unrecorded: 0,
+        reasons: {},
+        applied: 0,
+        holdMs: 0,
+        storedMs: 0,
+        applyMs: 0,
+        recordMs: 0,
+        lowerMs: 0,
+        replaceMs: 0,
+    };
 }
 /** Lowers `name` with its events recording, and makes its record. The unit
  * has a record when every table that grew is appendable and holds the
@@ -153,7 +215,9 @@ export function recordUnit(host, name, contents = null) {
     const marks = host.scalars.map((scalar) => scalar.mark());
     const counts = host.statementCounts();
     const recorder = new UnitEvents();
+    const lowering = performance.now();
     const { fn, poisoned } = host.lowerRecorded(name, recorder);
+    const lowerMs = performance.now() - lowering;
     const reasons = [];
     const grown = [];
     tables.forEach((table, index) => {
@@ -187,8 +251,8 @@ export function recordUnit(host, name, contents = null) {
         reasons.push("statements");
     if (now.other !== counts.other)
         reasons.push("diagnostics");
-    const text = fn === null ? null : encodeUnitFunction(fn);
-    if (fn !== null && text === null)
+    const encoded = fn === null ? null : encodeRecordFunction(fn);
+    if (fn !== null && encoded === null)
         reasons.push("encoding");
     const added = {};
     if (reasons.length === 0)
@@ -203,8 +267,8 @@ export function recordUnit(host, name, contents = null) {
     // A unit uses the entries its IR and new entries name: it reads them, and
     // a unit that changes one in place changes one it names.
     const texts = [];
-    if (text !== null)
-        texts.push(text);
+    if (typeof encoded?.fn === "string")
+        texts.push(encoded.fn);
     for (const index of grown) {
         const table = tables[index];
         const entries = added[table.name] ?? table.entries(before[index]);
@@ -212,7 +276,13 @@ export function recordUnit(host, name, contents = null) {
             for (const entry of entries)
                 texts.push(entry);
     }
+    // A unit names every entry it writes: the ones its function shows, and the
+    // ones it changed in place (UnitEvents.touch), which a function cannot.
     const names = quotedStrings(texts);
+    if (encoded?.strings != null)
+        namedStrings(encoded.strings, names);
+    for (const touched of recorder.touched)
+        names.add(touched);
     if (own !== null) {
         if (reasons.length === 0)
             tables.forEach((table, index) => {
@@ -249,30 +319,66 @@ export function recordUnit(host, name, contents = null) {
                 reads[table.name] = read;
         });
     if (reasons.length > 0)
-        return { record: null, reasons, fn, before };
+        return { record: null, reasons, fn, before, lowerMs, files: null };
     return {
-        record: { name, fn: text, poisoned, events, tables: added, reads },
+        record: { name, fn: encoded?.fn ?? null, poisoned, events, tables: added, reads },
         reasons,
         fn,
         before,
+        lowerMs,
+        files: encoded?.files ?? null,
     };
 }
-/** Lowers `wave` with every unit recorded. A unit with a record keeps the
- * results its record carries: the entries it added are taken out of their
- * tables and added again from the record, and its function is the one
- * decoded from the record. A unit without one keeps what lowering made.
+/** Lowers `wave` with every unit recorded. Without `reuse`, a unit with a
+ * record keeps the results its record carries: the entries it added are
+ * taken out of their tables and added again from the record, and its
+ * function is the one decoded from the record. Otherwise, and for a unit
+ * without a record, a unit keeps what lowering made.
  * Throws an InternalCompilerError when a unit changed the content of an
  * entry a table held when the unit started (JournalTable.contents), which
  * no record carries.
- * `lines`, when given, receives each unit's record as a JSON line. */
-export function lowerWaveRecorded(host, wave, stats, lines = null) {
+ * `lines`, when given, receives each unit's record as a JSON line. A unit
+ * whose stored record `reuse` gives applies it, and lowers when it does not
+ * apply. With `held`, the wave continues the content the waves before it
+ * left (HeldTables), and the check waits for HeldTables.check. */
+export function lowerWaveRecorded(host, wave, stats, lines = null, reuse = null, held = null) {
     stats.waves++;
     const tables = host.tables;
-    // Units keep the content of the entries that tables held when they
-    // started: a record carries a unit's new entries, not changes to others.
-    const contents = tables.map((table) => (table.contents === undefined ? null : table.contents()));
+    const contents = held ?? new HeldTables(tables, stats);
+    let started = 0;
     for (const name of wave) {
-        const unit = recordUnit(host, name, contents);
+        started = performance.now();
+        let stored = reuse?.stored(name) ?? null;
+        stats.storedMs += performance.now() - started;
+        let pending = {};
+        if (stored !== null && reuse.verify) {
+            // Verify mode lowers the units whose stored record would apply and
+            // checks the record they lower to.
+            if (recordApplies(host, stored.record).length === 0)
+                pending = pendingEntries(host, stored.record);
+            else
+                stored = null;
+        }
+        else if (stored !== null) {
+            started = performance.now();
+            const reasons = applyRecord(host, stored.record, stored.fn);
+            stats.applyMs += performance.now() - started;
+            if (reasons.length === 0) {
+                stats.units++;
+                stats.recorded++;
+                stats.applied++;
+                contents.extendTaken();
+                lines?.push(recordLine(stored.record, true));
+                reuse.note(name, stored.record, true, null);
+                continue;
+            }
+            for (const reason of reasons)
+                stats.reasons[`stored:${reason}`] = (stats.reasons[`stored:${reason}`] ?? 0) + 1;
+        }
+        started = performance.now();
+        const unit = recordUnit(host, name, contents.take());
+        stats.recordMs += performance.now() - started;
+        stats.lowerMs += unit.lowerMs;
         stats.units++;
         const record = unit.record;
         if (record === null) {
@@ -281,36 +387,170 @@ export function lowerWaveRecorded(host, wave, stats, lines = null) {
             for (const reason of unit.reasons)
                 stats.reasons[reason] = (stats.reasons[reason] ?? 0) + 1;
             host.store(name, unit.fn);
+            if (stored !== null && reuse.verify)
+                reuse.check(name, stored, pending, unit);
+            reuse?.note(name, null, false, null);
             continue;
         }
-        lines?.push(JSON.stringify(record));
+        lines?.push(recordLine(record));
         stats.recorded++;
-        tables.forEach((table, index) => {
-            const entries = record.tables[table.name];
-            if (entries === undefined)
-                return;
-            table.truncate(unit.before[index]);
-            table.apply(entries);
+        // The serial mode replaces a unit's results by its record's, so that the
+        // build's output is what applying the records gives. Reuse keeps what
+        // lowering made: its records apply in later builds, which verify mode
+        // and the edit-sequence tests check.
+        let fn = unit.fn;
+        if (reuse === null) {
+            const replacing = performance.now();
+            tables.forEach((table, index) => {
+                const entries = record.tables[table.name];
+                if (entries === undefined)
+                    return;
+                table.truncate(unit.before[index]);
+                table.apply(entries);
+            });
+            fn = record.fn === null ? null : decodeRecordFunction(record.fn);
+            stats.replaceMs += performance.now() - replacing;
+        }
+        host.store(name, fn);
+        if (stored !== null && reuse.verify)
+            reuse.check(name, stored, pending, unit);
+        reuse?.note(name, record, false, unit.files);
+    }
+    if (held === null)
+        contents.check();
+}
+/** The content of the tables' entries that recorded units start from
+ * (JournalTable.contents): units keep the content of the entries the
+ * tables held when they started, since a record carries a unit's new
+ * entries, not changes to others. Applying a stored record only adds
+ * entries, so the content is taken when the first unit lowers, and not at
+ * all when none does. Consecutive waves with nothing but checker prefetches
+ * between them share one: the content a wave leaves, as the check finds it,
+ * is the content the next one starts with. */
+export class HeldTables {
+    tables;
+    stats;
+    contents = null;
+    constructor(tables, stats) {
+        this.tables = tables;
+        this.stats = stats;
+    }
+    /** The held content, taken now when it was not yet. */
+    take() {
+        if (this.contents === null) {
+            const taken = performance.now();
+            this.contents = this.tables.map((table) => table.contents === undefined ? null : table.contents());
+            this.stats.holdMs += performance.now() - taken;
+        }
+        return this.contents;
+    }
+    /** Adds the entries past the held ones, when the content is taken. */
+    extendTaken() {
+        if (this.contents !== null)
+            for (const entry of this.contents)
+                entry?.extend();
+    }
+    /** Throws an InternalCompilerError when an entry a table held changed in
+     * place since the content was taken, other than by the units that keep
+     * their changes (HeldContents.refresh). */
+    check() {
+        const contents = this.contents;
+        if (contents === null)
+            return;
+        const started = performance.now();
+        this.tables.forEach((table, index) => {
+            const key = contents[index]?.changed() ?? null;
+            if (key !== null)
+                throw new InternalCompilerError(`a unit changed ${table.name} entry ${key} in place without naming it, which no unit record carries`);
         });
-        host.store(name, record.fn === null ? null : decodeUnitFunction(record.fn));
+        this.stats.holdMs += performance.now() - started;
     }
-    tables.forEach((table, index) => {
-        const key = contents[index]?.changed() ?? null;
-        if (key !== null)
-            throw new InternalCompilerError(`a unit changed ${table.name} entry ${key} in place without naming it, which no unit record carries`);
-    });
+}
+/** Applies `record` in place of lowering its unit, at the unit's turn, or
+ * returns why the unit has to lower instead. A record applies when every
+ * entry it adds agrees with what the tables hold, it names only known
+ * tables, every event resolves and every entry it reads is held with the
+ * same content. Applying adds its entries, keeps its function (`stored`'s,
+ * when the store keeps it apart from the record) and makes its events
+ * again. */
+export function applyRecord(host, record, stored) {
+    const reasons = recordApplies(host, record);
+    if (reasons.length > 0)
+        return reasons;
+    for (const table of host.tables) {
+        const entries = record.tables[table.name];
+        if (entries !== undefined)
+            table.apply(entries);
+    }
+    const fn = record.poisoned
+        ? null
+        : stored !== undefined
+            ? stored()
+            : record.fn === null
+                ? null
+                : decodeRecordFunction(record.fn);
+    host.keep(record.name, fn, record.events);
+    return [];
+}
+/** The entries `record` adds that the tables do not hold yet, by table
+ * name: what applying it would add at its unit's turn. */
+export function pendingEntries(host, record) {
+    const out = {};
+    for (const table of host.tables) {
+        const entries = record.tables[table.name];
+        if (entries === undefined)
+            continue;
+        const pending = entries.filter((entry) => table.holds?.(entry) !== true);
+        if (pending.length > 0)
+            out[table.name] = pending;
+    }
+    return out;
+}
+/** Why `record` does not apply at its unit's turn (applyRecord), or no
+ * reasons when it does. */
+export function recordApplies(host, record) {
+    const tables = host.tables;
+    for (const table of tables) {
+        const entries = record.tables[table.name];
+        if (entries === undefined)
+            continue;
+        if (table.agrees === undefined || table.apply === undefined || !table.agrees(entries))
+            return [`disagrees:${table.name}`];
+    }
+    for (const table of Object.keys(record.tables))
+        if (!tables.some((held) => held.name === table))
+            return [`unknown table:${table}`];
+    for (const event of record.events)
+        if (!host.resolves(event))
+            return [`event:${event.kind}`];
+    for (const table of tables) {
+        const reads = record.reads[table.name];
+        if (reads === undefined)
+            continue;
+        if (table.digest === undefined)
+            return [`unknown read:${table.name}`];
+        const size = table.size();
+        for (const [key, digest] of reads)
+            if (table.digest(key, size) !== digest)
+                return [`read:${table.name}`];
+    }
+    for (const table of Object.keys(record.reads))
+        if (!tables.some((held) => held.name === table))
+            return [`unknown read:${table}`];
+    return [];
 }
 /** The content of a keyed table's entries (JournalTable.contents): each
- * entry's encoded value, and the order of the keys, since a table only adds
- * entries at the end. An entry is found by its key, or by the name its value
- * holds (`valueName`): a unit names the helper it got, not the key it was
- * looked up by. `text` answers undefined for a key the table does not
+ * entry's content (`text`: its encoded value, or the value itself when it
+ * is a string or a number), and the order of the keys, since a table only
+ * adds entries at the end. An entry is found by its key, or by the name its
+ * value holds (`valueName`): a unit names the helper it got, not the key it
+ * was looked up by. `text` answers undefined for a key the table does not
  * hold. */
 function keyedContents(size, keys, text, valueName) {
     const texts = new Map();
     let order = [];
     const byName = new Map();
-    const differs = (key) => text(key) !== texts.get(key);
+    const differs = (key) => !Object.is(text(key), texts.get(key));
     const keyOf = (name) => (texts.has(name) ? name : byName.get(name));
     const held = {
         extend: () => {
@@ -371,11 +611,11 @@ function keyedContents(size, keys, text, valueName) {
 /** An appendable table over a Map from strings to strings, such as helper
  * names by key. */
 export function mapTable(name, map) {
-    return codecTable(name, map, (value) => JSON.stringify(value), (text) => JSON.parse(text), (value) => value);
+    return codecTable(name, map, (value) => JSON.stringify(value), (text) => JSON.parse(text), (value) => value, (value) => value);
 }
 /** An appendable table over a Map from strings to numbers. */
 export function countTable(name, map) {
-    return codecTable(name, map, (value) => String(value), (text) => Number(text));
+    return codecTable(name, map, (value) => String(value), (text) => Number(text), null, (value) => value);
 }
 /** An appendable table over a Map from strings to IR data, encoded as
  * JSON. JSON writes a number that is not finite as null, so a value whose
@@ -389,8 +629,11 @@ export function irDataTable(name, map) {
 /** An appendable table over a Map from strings to values that `encode`
  * writes, or answers null for, and `decode` reads back. An entry is its key
  * as a JSON string, a line break and its value: a JSON string holds no line
- * break. A value that does not encode leaves its unit without a record. */
-export function codecTable(name, map, encode, decode, nameOf = null) {
+ * break. A value that does not encode leaves its unit without a record.
+ * `content` is what held contents compare (JournalTable.contents): the
+ * encoding, or the value itself for a table of strings or numbers, which
+ * only change by being replaced. */
+export function codecTable(name, map, encode, decode, nameOf = null, content = encode) {
     return {
         name,
         size: () => map.size,
@@ -407,6 +650,12 @@ export function codecTable(name, map, encode, decode, nameOf = null) {
             }
             return out;
         },
+        agrees: (entries) => entries.every((entry) => {
+            const split = entry.indexOf("\n");
+            const held = map.get(JSON.parse(entry.slice(0, split)));
+            return held === undefined || encode(held) === entry.slice(split + 1);
+        }),
+        holds: (entry) => map.has(JSON.parse(entry.slice(0, entry.indexOf("\n")))),
         apply: (entries) => {
             for (const entry of entries) {
                 const split = entry.indexOf("\n");
@@ -422,7 +671,7 @@ export function codecTable(name, map, encode, decode, nameOf = null) {
         },
         contents: () => keyedContents(() => map.size, () => map.keys(), (key) => {
             const value = map.get(key);
-            return value === undefined ? undefined : encode(value);
+            return value === undefined ? undefined : content(value);
         }, (key) => {
             const value = map.get(key);
             return value === undefined || nameOf === null ? null : nameOf(value);
@@ -435,6 +684,8 @@ export function setTable(name, set) {
         name,
         size: () => set.size,
         entries: (from) => [...set].slice(from),
+        agrees: () => true,
+        holds: (entry) => set.has(entry),
         apply: (entries) => {
             for (const entry of entries)
                 set.add(entry);
@@ -497,6 +748,14 @@ export function functionTable(name, functions, has, changed = () => { }) {
             }
             return out;
         },
+        agrees: (entries) => entries.every((entry) => {
+            const fn = decodeUnitFunction(entry);
+            if (!has(fn.name))
+                return true;
+            const index = indexOf(fn.name);
+            return index !== undefined && encodeUnitFunction(functions[index]) === entry;
+        }),
+        holds: (entry) => has(decodeUnitFunction(entry).name),
         apply: (entries) => {
             for (const entry of entries) {
                 const fn = decodeUnitFunction(entry);
@@ -510,15 +769,16 @@ export function functionTable(name, functions, has, changed = () => { }) {
                 functions.pop();
             changed();
         },
+        // Held functions are their content digests (heldFunction).
         contents: () => {
             const texts = [];
             const indexes = new Map();
-            const differs = (index) => index < functions.length && encodeUnitFunction(functions[index]) !== texts[index];
+            const differs = (index) => index < functions.length && heldFunction(functions[index]) !== texts[index];
             const held = {
                 extend: () => {
                     for (let index = texts.length; index < functions.length; index++) {
                         const fn = functions[index];
-                        texts.push(encodeUnitFunction(fn));
+                        texts.push(heldFunction(fn));
                         indexes.set(fn.name, index);
                     }
                 },
@@ -540,7 +800,7 @@ export function functionTable(name, functions, has, changed = () => { }) {
                     for (const key of keys) {
                         const index = indexes.get(key);
                         if (index !== undefined && index < functions.length)
-                            texts[index] = encodeUnitFunction(functions[index]);
+                            texts[index] = heldFunction(functions[index]);
                     }
                 },
             };
@@ -549,6 +809,17 @@ export function functionTable(name, functions, has, changed = () => { }) {
         },
     };
 }
+/** The content a function table holds of `fn` (HeldContents): its digest
+ * (functionDigest), or its entry encoding when the binary encoder cannot
+ * write it. */
+function heldFunction(fn) {
+    try {
+        return functionDigest(fn);
+    }
+    catch {
+        return encodeUnitFunction(fn);
+    }
+}
 /** A table whose growth leaves the unit without a record. */
 export function checkedTable(name, size) {
     return { name, size };