c64-kb 0.17.0

package/dist/graph/extract.js

@@ -0,0 +1,43 @@
+/**
+ * Extract graph entities from knowledge-base markdown documents.
+ *
+ * A page declares its type with a `<!-- doc-type: … -->` marker; each type
+ * has one parser under src/graph/extract/, following the matching
+ * docs/CONVENTIONS-*.md. This file dispatches on the marker and re-exports
+ * the parsers' public names so importers keep one path.
+ */
+import { parseArchetypeDoc } from "./extract/archetype.js";
+import { parseFailureDoc } from "./extract/failure.js";
+import { parseGameDesignDoc } from "./extract/game-design.js";
+import { parseHardwareDoc } from "./extract/hardware.js";
+import { parsePitfallDoc } from "./extract/pitfall.js";
+import { parseRecipeDoc } from "./extract/recipe.js";
+import { parseTechniqueDoc } from "./extract/technique.js";
+import { parseFormatDoc, parseToolchainDoc } from "./extract/toolchain.js";
+export { COST_BASIS_WORDS, COST_VOCABULARY, DEMAND_VOCABULARY } from "./extract/vocabulary.js";
+export { parseRasterBand, rasterBandsOverlap } from "./extract/raster-band.js";
+/**
+ * Marker -> parser, in the order the markers are tested. A page carrying
+ * more than one marker is parsed by the first that matches.
+ */
+const PARSERS = [
+    ["<!-- doc-type: hardware-reference -->", parseHardwareDoc],
+    ["<!-- doc-type: toolchain-reference -->", parseToolchainDoc],
+    ["<!-- doc-type: format-reference -->", parseFormatDoc],
+    ["<!-- doc-type: recipe -->", parseRecipeDoc],
+    ["<!-- doc-type: technique-reference -->", parseTechniqueDoc],
+    ["<!-- doc-type: pitfall-reference -->", parsePitfallDoc],
+    ["<!-- doc-type: archetype-reference -->", parseArchetypeDoc],
+    ["<!-- doc-type: failure-reference -->", parseFailureDoc],
+    ["<!-- doc-type: game-design -->", parseGameDesignDoc],
+];
+export function extractGraphEntities(content, sourcePath) {
+    // CONVENTIONS-*.md docs intentionally carry doc-type markers to explain them,
+    // but their example headings should not become graph entities. Skip by basename.
+    const basename = sourcePath.replace(/^.*\//, "");
+    if (basename.startsWith("CONVENTIONS-"))
+        return [];
+    const entry = PARSERS.find(([marker]) => content.includes(marker));
+    return entry ? entry[1](content, sourcePath) : [];
+}
+//# sourceMappingURL=extract.js.map

package/dist/graph/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/graph/extract.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAI3E,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC/F,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE/E;;;GAGG;AACH,MAAM,OAAO,GAA6D;IACxE,CAAC,uCAAuC,EAAE,gBAAgB,CAAC;IAC3D,CAAC,wCAAwC,EAAE,iBAAiB,CAAC;IAC7D,CAAC,qCAAqC,EAAE,cAAc,CAAC;IACvD,CAAC,2BAA2B,EAAE,cAAc,CAAC;IAC7C,CAAC,wCAAwC,EAAE,iBAAiB,CAAC;IAC7D,CAAC,sCAAsC,EAAE,eAAe,CAAC;IACzD,CAAC,wCAAwC,EAAE,iBAAiB,CAAC;IAC7D,CAAC,sCAAsC,EAAE,eAAe,CAAC;IACzD,CAAC,gCAAgC,EAAE,kBAAkB,CAAC;CACvD,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,UAAkB;IACtE,8EAA8E;IAC9E,iFAAiF;IACjF,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC;QAAE,OAAO,EAAE,CAAC;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACpD,CAAC"}

package/dist/graph/kernal-clobbers.js

@@ -0,0 +1,64 @@
+// The `**Clobbers zero page:**` line under a KERNAL routine H3 (schema 26):
+// the zero-page bytes the routine may or must write, and where the set came
+// from. Written by scripts/kernal-zp-walk.ts (bound `may`, a static ROM
+// walk); a `must` line records what a VICE store trace saw. The extractor
+// turns each line into a CLOBBERS_ZP edge to the zero_page HardwareUnit.
+//
+//   **Clobbers zero page:** $B8-$BA (may; ROM walk from $FFBA, power-on vectors)
+//   **Clobbers zero page:** none (may; ROM walk from $FFED, power-on vectors)
+//   **Clobbers zero page:** $B8-$BA (must; VICE x64sc store trace, SETLFS 2,8,2)
+import { group } from "./extract/common.js";
+export const CLOBBERS_LABEL = "**Clobbers zero page:**";
+export const CLOBBERS_LINE = /^\*\*Clobbers zero page:\*\*\s*(.+?)\s*$/gm;
+const hex2 = (n) => n.toString(16).toUpperCase().padStart(2, "0");
+/** `$90-$9A, $B7` from merged ranges; `none` for an empty set. */
+export function formatClobberRanges(ranges) {
+    if (ranges.length === 0)
+        return "none";
+    return ranges.map(([a, b]) => (a === b ? `$${hex2(a)}` : `$${hex2(a)}-$${hex2(b)}`)).join(", ");
+}
+/** The whole line value, as the walk script writes it. */
+export function formatClobbers(c) {
+    return `${formatClobberRanges(c.ranges)} (${c.bound}; ${c.basis})`;
+}
+/** Merge and sort [first, last] pairs. */
+function merge(ranges) {
+    const sorted = [...ranges].sort((x, y) => x[0] - y[0]);
+    const out = [];
+    for (const r of sorted) {
+        const prev = out.at(-1);
+        if (prev && r[0] <= prev[1] + 1)
+            prev[1] = Math.max(prev[1], r[1]);
+        else
+            out.push([r[0], r[1]]);
+    }
+    return out;
+}
+/**
+ * Parse a line value. Zero page here is $00-$FF: the KERNAL writes the 6510
+ * port at $00-$01 too (IOINIT, the tape motor), and a reader of the line
+ * should see it, though no Claims line can name those two bytes.
+ */
+export function parseClobbers(value) {
+    const m = /^(.*?)\s*\((may|must);\s*(.+)\)$/.exec(value.trim());
+    if (!m)
+        return { error: `"${value}" is not "<bytes> (may|must; <basis>)"` };
+    const set = group(m, 1).trim();
+    const bound = group(m, 2);
+    const basis = group(m, 3).trim();
+    if (set === "none")
+        return { ranges: [], bound, basis };
+    const ranges = [];
+    for (const part of set.split(",").map((s) => s.trim())) {
+        const r = /^\$([0-9A-Fa-f]{2})(?:-\$([0-9A-Fa-f]{2}))?$/.exec(part);
+        if (!r)
+            return { error: `"${part}" is not a zero-page byte $XX or range $XX-$YY` };
+        const first = parseInt(group(r, 1), 16);
+        const last = r[2] ? parseInt(r[2], 16) : first;
+        if (first > last)
+            return { error: `range ${part} runs backwards` };
+        ranges.push([first, last]);
+    }
+    return { ranges: merge(ranges), bound, basis };
+}
+//# sourceMappingURL=kernal-clobbers.js.map

package/dist/graph/kernal-clobbers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"kernal-clobbers.js","sourceRoot":"","sources":["../../src/graph/kernal-clobbers.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,4EAA4E;AAC5E,wEAAwE;AACxE,0EAA0E;AAC1E,yEAAyE;AACzE,EAAE;AACF,iFAAiF;AACjF,8EAA8E;AAC9E,iFAAiF;AAEjF,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAY5C,MAAM,CAAC,MAAM,cAAc,GAAG,yBAAyB,CAAC;AACxD,MAAM,CAAC,MAAM,aAAa,GAAG,4CAA4C,CAAC;AAE1E,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAElF,kEAAkE;AAClE,MAAM,UAAU,mBAAmB,CAAC,MAAmC;IACrE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACvC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,cAAc,CAAC,CAAiB;IAC9C,OAAO,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC;AACrE,CAAC;AAED,0CAA0C;AAC1C,SAAS,KAAK,CAAC,MAA0B;IACvC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;YAC9D,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,MAAM,CAAC,GAAG,kCAAkC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,KAAK,wCAAwC,EAAE,CAAC;IAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC,CAAiB,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACjC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACxD,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACvD,MAAM,CAAC,GAAG,8CAA8C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,IAAI,IAAI,gDAAgD,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,IAAI,KAAK,GAAG,IAAI;YAAE,OAAO,EAAE,KAAK,EAAE,SAAS,IAAI,iBAAiB,EAAE,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACjD,CAAC"}

package/dist/graph/machine-variants.js

@@ -0,0 +1,208 @@
+/**
+ * MachineVariant (schema 29): the C64 models VICE x64sc 3.10 runs here, and
+ * the VERIFIED_ON edges docs/recipes/runs.json implies. The chips of each
+ * model were read from VICE itself: `x64sc -default -model <m> -dumpconfig`
+ * (resources VICIIModel, SidModel, CIA1Model), each VICIIModel number
+ * named by dumping `-VICIImodel <chip>`. The dump's KernalName and
+ * KernalRev name the KERNAL image (901227-01 on oldntsc, -02 on c64old,
+ * -03 elsewhere); the seeds do not carry it. Lines per frame were
+ * measured by recipes/oscar64/pal-ntsc-detect.md on all seven models;
+ * cycles per line are from docs/hardware/pal-ntsc-reference.md.
+ *
+ * `x64sc -default` with no `-model` dumps a configuration identical to
+ * `-model c64c` (8565, 8580, 8521), not `c64` (6569, 6581, 6526). The
+ * cia-revision-detect listing confirms it: with no -ciamodel flag it
+ * reads the new CIA on the default machine and the old one on -model c64.
+ * So a runs.json "pal" run is a c64c run.
+ */
+import { z } from "zod";
+export const MACHINE_VARIANTS = [
+    {
+        name: "c64",
+        vic: "6569",
+        sid: "6581",
+        cia: "6526",
+        cycles_per_line: 63,
+        lines: 312,
+        region: "PAL",
+        vice_default: false,
+    },
+    {
+        name: "c64c",
+        vic: "8565",
+        sid: "8580",
+        cia: "8521",
+        cycles_per_line: 63,
+        lines: 312,
+        region: "PAL",
+        vice_default: true,
+    },
+    {
+        name: "c64old",
+        vic: "6569R1",
+        sid: "6581",
+        cia: "6526",
+        cycles_per_line: 63,
+        lines: 312,
+        region: "PAL",
+        vice_default: false,
+    },
+    {
+        name: "ntsc",
+        vic: "6567R8",
+        sid: "6581",
+        cia: "6526",
+        cycles_per_line: 65,
+        lines: 263,
+        region: "NTSC",
+        vice_default: false,
+    },
+    {
+        name: "newntsc",
+        vic: "8562",
+        sid: "8580",
+        cia: "8521",
+        cycles_per_line: 65,
+        lines: 263,
+        region: "NTSC",
+        vice_default: false,
+    },
+    {
+        name: "oldntsc",
+        vic: "6567R56A",
+        sid: "6581",
+        cia: "6526",
+        cycles_per_line: 64,
+        lines: 262,
+        region: "NTSC",
+        vice_default: false,
+    },
+    {
+        name: "drean",
+        vic: "6572",
+        sid: "6581",
+        cia: "6526",
+        cycles_per_line: 65,
+        lines: 312,
+        region: "PAL-N",
+        vice_default: false,
+    },
+];
+/**
+ * runs.json model words to the variant VICE runs for them
+ * (scripts/verify-recipes.ts MODEL_FLAG): "pal" adds no -model flag, so it
+ * is VICE's default machine.
+ */
+export const RUNS_MODEL_VARIANT = {
+    pal: "c64c",
+    ntsc: "ntsc",
+    oldntsc: "oldntsc",
+    drean: "drean",
+};
+const RunEntry = z.object({
+    cycles: z.number().int().positive().optional(),
+    models: z.array(z.string()).optional(),
+    flags: z.array(z.string()).optional(),
+    shots: z.record(z.string(), z.string()).optional(),
+    cartridge: z.object({ runs: z.number().int().positive().optional() }).loose().optional(),
+    /** verify:recipes does not run this page (it says why); no edge. */
+    skip: z.string().optional(),
+});
+const Manifest = z.record(z.string(), z.union([z.string(), RunEntry.loose()]));
+const DEFAULT_CYCLES = 8_000_000;
+/** verify-recipes' default shot path for boot 1 of a model. */
+function defaultShot(stem, model) {
+    return `screenshots/${stem}${model === "pal" ? "" : `-${model}`}.png`;
+}
+/** One page's edges, pushing problems onto the caller's lists. */
+function pageEdges(page, entry, shotExists, out) {
+    const [toolchain = "", stem = ""] = page.split("/");
+    for (const model of entry?.models ?? ["pal"]) {
+        const variant = RUNS_MODEL_VARIANT[model];
+        if (!variant) {
+            out.unknownModels.push(`${page}: ${model}`);
+            continue;
+        }
+        const shot = entry?.shots?.[model] ?? defaultShot(stem, model);
+        if (!shotExists(toolchain, shot)) {
+            out.missingShots.push(`${page}: ${shot}`);
+            continue;
+        }
+        out.edges.push({
+            source_doc: `recipes/${page}.md`,
+            variant,
+            model,
+            cycles: entry?.cycles ?? DEFAULT_CYCLES,
+            shot,
+            flags: (entry?.flags ?? []).join(" "),
+            pinned: entry !== undefined,
+        });
+    }
+}
+/**
+ * The VERIFIED_ON edges for every recipe page, as verify:recipes runs them:
+ * the runs.json entry when there is one, else PAL at 8,000,000 cycles. An
+ * edge is emitted only when its committed screenshot exists, because that
+ * picture is what the run is compared with. A page whose entry has `skip`
+ * is not run by verify:recipes and gets no edge. A model word with no
+ * variant is reported in `unknownModels`.
+ */
+export function verifiedOnEdges(opts) {
+    const manifest = Manifest.parse(JSON.parse(opts.manifestJson));
+    const out = { edges: [], unknownModels: [], missingShots: [] };
+    for (const page of opts.pages) {
+        const raw = manifest[page];
+        if (typeof raw === "object" && raw.skip)
+            continue;
+        pageEdges(page, typeof raw === "object" ? raw : undefined, opts.shotExists, out);
+    }
+    return out;
+}
+// Values from `x64sc -help` (VICE 3.10): -ciamodel 0 = old 6526, 1 = new
+// 8521; -sidmodel 0 = 6581, 1 = 8580, 2 = 8580 + digiboost; -VICIImodel
+// takes a chip name. The VIC names map to the seed spellings above.
+const CIA_BY_VALUE = { "0": "6526", "1": "8521" };
+const SID_BY_VALUE = { "0": "6581", "1": "8580", "2": "8580 + digiboost" };
+const VIC_BY_NAME = {
+    "6569": "6569",
+    "6569r1": "6569R1",
+    "8565": "8565",
+    "6567": "6567R8",
+    "8562": "8562",
+    "6567r56a": "6567R56A",
+    "6572": "6572",
+};
+/** Each chip flag: the chips it sets, the label it reports, and its value table. */
+const CHIP_FLAGS = {
+    "-ciamodel": { keys: ["cia1", "cia2"], label: "CIA", value: (v) => CIA_BY_VALUE[v] },
+    "-cia1model": { keys: ["cia1"], label: "CIA 1", value: (v) => CIA_BY_VALUE[v] },
+    "-cia2model": { keys: ["cia2"], label: "CIA 2", value: (v) => CIA_BY_VALUE[v] },
+    "-sidmodel": { keys: ["sid"], label: "SID", value: (v) => SID_BY_VALUE[v] },
+    "-viciimodel": { keys: ["vic"], label: "VIC-II", value: (v) => VIC_BY_NAME[v.toLowerCase()] },
+};
+export function effectiveChips(v, flags) {
+    const chips = { vic: v.vic, sid: v.sid, cia1: v.cia, cia2: v.cia };
+    const overrides = [];
+    const words = flags.split(/\s+/).filter(Boolean);
+    words.forEach((word, i) => {
+        const w = word.toLowerCase();
+        const val = words[i + 1] ?? "";
+        if (w === "-model") {
+            overrides.push(`machine replaced (${word} ${val}): the chips listed are not the run's`);
+            return;
+        }
+        const f = CHIP_FLAGS[w];
+        if (!f)
+            return;
+        const next = f.value(val) ?? "?";
+        const before = [...new Set(f.keys.map((k) => chips[k]))].join("/");
+        if (before === next)
+            return;
+        overrides.push(`${f.label} ${before} -> ${next} (${word} ${val})`);
+        for (const k of f.keys)
+            chips[k] = next;
+    });
+    const cia = chips.cia1 === chips.cia2 ? chips.cia1 : `${chips.cia1}/${chips.cia2}`;
+    return { vic: chips.vic, sid: chips.sid, cia, overrides };
+}
+//# sourceMappingURL=machine-variants.js.map

package/dist/graph/machine-variants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"machine-variants.js","sourceRoot":"","sources":["../../src/graph/machine-variants.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgBxB,MAAM,CAAC,MAAM,gBAAgB,GAA8B;IACzD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,KAAK;QACb,YAAY,EAAE,KAAK;KACpB;IACD;QACE,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,KAAK;QACb,YAAY,EAAE,IAAI;KACnB;IACD;QACE,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,QAAQ;QACb,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,KAAK;QACb,YAAY,EAAE,KAAK;KACpB;IACD;QACE,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,QAAQ;QACb,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,MAAM;QACd,YAAY,EAAE,KAAK;KACpB;IACD;QACE,IAAI,EAAE,SAAS;QACf,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,MAAM;QACd,YAAY,EAAE,KAAK;KACpB;IACD;QACE,IAAI,EAAE,SAAS;QACf,GAAG,EAAE,UAAU;QACf,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,MAAM;QACd,YAAY,EAAE,KAAK;KACpB;IACD;QACE,IAAI,EAAE,OAAO;QACb,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,MAAM;QACX,eAAe,EAAE,EAAE;QACnB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,OAAO;QACf,YAAY,EAAE,KAAK;KACpB;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAqC;IAClE,GAAG,EAAE,MAAM;IACX,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;CACf,CAAC;AAkBF,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IACxB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE;IACxF,oEAAoE;IACpE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AACH,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AAE/E,MAAM,cAAc,GAAG,SAAS,CAAC;AAEjC,+DAA+D;AAC/D,SAAS,WAAW,CAAC,IAAY,EAAE,KAAa;IAC9C,OAAO,eAAe,IAAI,GAAG,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,MAAM,CAAC;AACxE,CAAC;AAID,kEAAkE;AAClE,SAAS,SAAS,CAChB,IAAY,EACZ,KAAwB,EACxB,UAAwD,EACxD,GAA6E;IAE7E,MAAM,CAAC,SAAS,GAAG,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC;YAC5C,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC;YAC1C,SAAS;QACX,CAAC;QACD,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YACb,UAAU,EAAE,WAAW,IAAI,KAAK;YAChC,OAAO;YACP,KAAK;YACL,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,cAAc;YACvC,IAAI;YACJ,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YACrC,MAAM,EAAE,KAAK,KAAK,SAAS;SAC5B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,IAK/B;IACC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,EAAE,KAAK,EAAE,EAAkB,EAAE,aAAa,EAAE,EAAc,EAAE,YAAY,EAAE,EAAc,EAAE,CAAC;IACvG,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI;YAAE,SAAS;QAClD,SAAS,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAWD,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,MAAM,YAAY,GAAqC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACpF,MAAM,YAAY,GAAqC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC;AAC7G,MAAM,WAAW,GAAqC;IACpD,MAAM,EAAE,MAAM;IACd,QAAQ,EAAE,QAAQ;IAClB,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,MAAM;IACd,UAAU,EAAE,UAAU;IACtB,MAAM,EAAE,MAAM;CACf,CAAC;AAiBF,oFAAoF;AACpF,MAAM,UAAU,GAEZ;IACF,WAAW,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE;IACpF,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE;IAC/E,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE;IAC/E,WAAW,EAAE,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE;IAC3E,aAAa,EAAE,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE;CAC9F,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,CAA4C,EAAE,KAAa;IACxF,MAAM,KAAK,GAAU,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IAC1E,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC;YACnB,SAAS,CAAC,IAAI,CAAC,qBAAqB,IAAI,IAAI,GAAG,uCAAuC,CAAC,CAAC;YACxF,OAAO;QACT,CAAC;QACD,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QACjC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnE,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO;QAC5B,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,MAAM,OAAO,IAAI,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC;QACnE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI;YAAE,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAC1C,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;IACnF,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC5D,CAAC"}

package/dist/ingest/files.js

@@ -0,0 +1,99 @@
+/**
+ * Batch ingest's files: the docs walk, the per-file content hashes that
+ * make a run incremental, the ingest log, and the BM25 vocab fitted over
+ * the whole corpus.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+import { config } from "../config.js";
+import { BM25_VOCAB_FILE, BM25Encoder, loadBM25Vocab } from "../services/bm25.js";
+export const HASH_FILE = path.resolve(config.analytics.dbPath, "../ingest-hashes.json");
+const LOG_FILE = path.resolve(config.analytics.dbPath, "../ingest.log");
+export const VOCAB_FILE = BM25_VOCAB_FILE;
+const Hashes = z.record(z.string(), z.string());
+/** Content hash per docs-relative path from the last run; empty when there is none or it is unreadable. */
+export function loadHashes() {
+    if (!fs.existsSync(HASH_FILE))
+        return {};
+    try {
+        const parsed = Hashes.safeParse(JSON.parse(fs.readFileSync(HASH_FILE, "utf-8")));
+        if (parsed.success)
+            return parsed.data;
+        console.error(`[ingest] ${HASH_FILE} is not a hash map — every file will be re-ingested`);
+    }
+    catch (err) {
+        console.error(`[ingest] cannot read ${HASH_FILE}: ${err instanceof Error ? err.message : String(err)} — every file will be re-ingested`);
+    }
+    return {};
+}
+export function saveHashes(hashes) {
+    fs.mkdirSync(path.dirname(HASH_FILE), { recursive: true });
+    fs.writeFileSync(HASH_FILE, JSON.stringify(hashes, null, 2));
+}
+export function log(msg) {
+    fs.mkdirSync(path.dirname(LOG_FILE), { recursive: true });
+    fs.appendFileSync(LOG_FILE, `${new Date().toISOString()} ${msg}\n`);
+}
+/**
+ * Load the BM25 vocab from disk if present; otherwise (or with
+ * `forceRefit`) fit a fresh encoder over the full corpus, so all sparse
+ * vectors share the same vocabulary indices, and persist it.
+ */
+export function loadOrFitBM25(corpus, forceRefit) {
+    const loaded = forceRefit ? null : loadBM25Vocab(VOCAB_FILE);
+    if (loaded)
+        return { encoder: loaded, fitted: false };
+    const encoder = new BM25Encoder();
+    encoder.fit(corpus);
+    fs.mkdirSync(path.dirname(VOCAB_FILE), { recursive: true });
+    fs.writeFileSync(VOCAB_FILE, JSON.stringify(encoder.toJSON()));
+    return { encoder, fitted: true };
+}
+/**
+ * Walk priority is now ONLY load-bearing for FileFormat description authority:
+ * docs/formats/c64-file-formats.md must land first so addFileFormat's
+ * ON-CREATE-only description set wins over later per-toolchain `.FORMAT`
+ * blurbs. Everything else used to depend on this order for edge correctness,
+ * but the two-pass linker (P0-3) defers all edges to pass 2, where every
+ * node already exists — so node-creation order no longer affects edge
+ * outcomes. The hardware/toolchains/runtime/techniques/recipes ordering
+ * remains for log readability + the FileFormat race.
+ */
+const WALK_PRIORITY = [
+    "formats/", // FileFormat catalog: descriptions land first (ON CREATE SET wins). LOAD-BEARING.
+    "hardware/", // Register / KernalRoutine / MemoryRegion nodes (cosmetic ordering only)
+    "toolchains/", // Tool nodes (cosmetic)
+    "runtime/", // More Tool nodes — vice, vice-mcp, sim6502 (cosmetic)
+    "techniques/", // Technique nodes (cosmetic — edges resolve in pass 2)
+    "recipes/", // Recipe nodes (cosmetic — edges resolve in pass 2)
+];
+function walkRank(rel) {
+    const i = WALK_PRIORITY.findIndex((p) => rel.startsWith(p));
+    return i === -1 ? WALK_PRIORITY.length : i;
+}
+/**
+ * Recursively walk a directory and return all *.md paths relative to root,
+ * with forward slashes, in walk-priority order. Skips non-knowledge
+ * subtrees (specs and plans).
+ */
+export function findMarkdown(root) {
+    if (!fs.existsSync(root))
+        return [];
+    // Node's native recursive readdir: every returned name is
+    // filesystem-derived, so no path is built from user-influenced segments.
+    const entries = fs.readdirSync(root, { withFileTypes: true, recursive: true });
+    const results = [];
+    for (const entry of entries) {
+        if (!entry.isFile() || !entry.name.endsWith(".md"))
+            continue;
+        // entry.parentPath is the directory the file was discovered in.
+        const parent = entry.parentPath;
+        const rel = parent === root ? entry.name : `${parent.slice(root.length).replace(/^[/\\]+/, "")}/${entry.name}`;
+        if (rel.startsWith("superpowers/") || rel.startsWith("superpowers\\"))
+            continue;
+        results.push(rel.replace(/\\/g, "/"));
+    }
+    return results.sort((a, b) => walkRank(a) - walkRank(b) || a.localeCompare(b));
+}
+//# sourceMappingURL=files.js.map

package/dist/ingest/files.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/ingest/files.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElF,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;AACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,UAAU,GAAG,eAAe,CAAC;AAE1C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AAEhD,2GAA2G;AAC3G,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACjF,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,MAAM,CAAC,IAAI,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC,YAAY,SAAS,qDAAqD,CAAC,CAAC;IAC5F,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CACX,wBAAwB,SAAS,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,mCAAmC,CAC1H,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAA8B;IACvD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,GAAW;IAC7B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;AACtE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAgB,EAChB,UAAmB;IAEnB,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,MAAM;QAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AACnC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,aAAa,GAAsB;IACvC,UAAU,EAAE,kFAAkF;IAC9F,WAAW,EAAE,yEAAyE;IACtF,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,uDAAuD;IACnE,aAAa,EAAE,uDAAuD;IACtE,UAAU,EAAE,oDAAoD;CACjE,CAAC;AAEF,SAAS,QAAQ,CAAC,GAAW;IAC3B,MAAM,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,0DAA0D;IAC1D,yEAAyE;IACzE,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QAC7D,gEAAgE;QAChE,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;QAChC,MAAM,GAAG,GACP,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACrG,IAAI,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;YAAE,SAAS;QAChF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC"}

package/dist/ingest/passes.js

@@ -0,0 +1,104 @@
+/**
+ * Batch ingest's two passes. Pass 1 writes each changed file's vectors and
+ * graph nodes and collects its edges; pass 2 applies every collected edge
+ * once all nodes across all files exist, so an edge never depends on the
+ * order files were walked in (FalkorDB's MERGE-stub fallback covers the
+ * rest).
+ */
+import { createHash } from "node:crypto";
+import { applyEdge, applyNode, isNodeEntity, recipeEdges, } from "../graph/apply.js";
+import { extractGraphEntities } from "../graph/extract.js";
+import { chunkMarkdown } from "../services/chunker.js";
+import { log, saveHashes } from "./files.js";
+import { replaceDocPoints } from "./points.js";
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function countNode(nodes, e) {
+    if (e.type === "pitfall")
+        nodes.pitfalls++;
+    else if (e.type === "crash_pattern")
+        nodes.crashPatterns++;
+    else if (e.type === "archetype")
+        nodes.archetypes++;
+    else if (e.type === "game_design")
+        nodes.gameDesigns++;
+}
+/** Create a file's nodes and queue its edges. Returns the number of nodes created. */
+async function applyFileEntities(ctx, file, content) {
+    let entityCount = 0;
+    for (const e of extractGraphEntities(content, file)) {
+        if (!isNodeEntity(e)) {
+            ctx.pending.push(e);
+            continue;
+        }
+        try {
+            await applyNode(ctx.falkor, e);
+            entityCount++;
+            countNode(ctx.nodes, e);
+            if (e.type === "recipe")
+                ctx.pending.push(...recipeEdges(e));
+        }
+        catch (err) {
+            console.warn(`  graph error for ${e.type} in ${file}: ${String(err)}`);
+        }
+    }
+    return entityCount;
+}
+/** Pass 1 for one file: skip it when unchanged, else replace its points and create its nodes. */
+export async function ingestFile(ctx, file, content) {
+    const contentHash = createHash("sha256").update(content).digest("hex");
+    if (!ctx.forceAll && ctx.hashes[file] === contentHash) {
+        ctx.nodes.skipped++;
+        return;
+    }
+    ctx.hashes[file] = contentHash;
+    saveHashes(ctx.hashes);
+    const chunks = chunkMarkdown(content, file);
+    if (chunks.length === 0)
+        return;
+    const points = await replaceDocPoints(ctx.qdrant, { source: file, chunks, bm25: ctx.bm25 });
+    ctx.nodes.chunks += points;
+    const entityCount = await applyFileEntities(ctx, file, content);
+    ctx.print(`  ${file}: ${points} chunks, ${entityCount} graph entities`);
+    log(`INGEST ${file} chunks=${points} graph=${entityCount}`);
+}
+// Connection-class errors abort pass 2 at once; per-row malformation is
+// logged and skipped. Ten consecutive failures also abort (catches FalkorDB
+// dropping mid-pass without a socket error).
+const FATAL_PATTERNS = [/ECONNREFUSED/i, /ECONNRESET/i, /Connection is closed/i, /Redis connection/i];
+const FAILURE_THRESHOLD = 10;
+/** Pass 2: apply every deferred edge, recording each outcome in `tally`. */
+export async function applyPendingEdges(falkor, pass) {
+    const { edges, tally, print } = pass;
+    print(`\nPass 2: applying ${edges.length} deferred edges`);
+    let edgeFailures = 0;
+    let consecutiveFailures = 0;
+    for (const edge of edges) {
+        try {
+            tally.record(edge, await applyEdge(falkor, edge));
+            consecutiveFailures = 0;
+        }
+        catch (err) {
+            edgeFailures++;
+            const message = errorMessage(err);
+            if (FATAL_PATTERNS.some((re) => re.test(message))) {
+                console.error(`[ingest] FATAL: connection-class error in pass 2: ${message}`);
+                throw err;
+            }
+            consecutiveFailures++;
+            if (consecutiveFailures >= FAILURE_THRESHOLD) {
+                console.error(`[ingest] FATAL: ${consecutiveFailures} consecutive edge failures — aborting`);
+                throw new Error(`Pass-2 edge ingest exceeded failure threshold (${FAILURE_THRESHOLD})`, {
+                    cause: err,
+                });
+            }
+            console.warn(`[ingest] edge failure (${edgeFailures} total): ${message}`);
+        }
+    }
+    if (edgeFailures > 0) {
+        console.warn(`\nPass 2: ${edgeFailures} edges failed to land (graph state may be incomplete)`);
+        log(`PASS2_FAILURES ${edgeFailures}`);
+    }
+}
+//# sourceMappingURL=passes.js.map

package/dist/ingest/passes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"passes.js","sourceRoot":"","sources":["../../src/ingest/passes.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,SAAS,EACT,SAAS,EACT,YAAY,EACZ,WAAW,GAGZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAGvD,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAgB/C,SAAS,YAAY,CAAC,GAAY;IAChC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,SAAS,CAAC,KAAgB,EAAE,CAAa;IAChD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;SACtC,IAAI,CAAC,CAAC,IAAI,KAAK,eAAe;QAAE,KAAK,CAAC,aAAa,EAAE,CAAC;SACtD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW;QAAE,KAAK,CAAC,UAAU,EAAE,CAAC;SAC/C,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,KAAK,CAAC,WAAW,EAAE,CAAC;AACzD,CAAC;AAED,sFAAsF;AACtF,KAAK,UAAU,iBAAiB,CAAC,GAAiB,EAAE,IAAY,EAAE,OAAe;IAC/E,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC/B,WAAW,EAAE,CAAC;YACd,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;gBAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,IAAI,OAAO,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,iGAAiG;AACjG,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAiB,EAAE,IAAY,EAAE,OAAe;IAC/E,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACtD,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC;IAC/B,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAEvB,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAChC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5F,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;IAE3B,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAChE,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,MAAM,YAAY,WAAW,iBAAiB,CAAC,CAAC;IACxE,GAAG,CAAC,UAAU,IAAI,WAAW,MAAM,UAAU,WAAW,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,wEAAwE;AACxE,4EAA4E;AAC5E,6CAA6C;AAC7C,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,aAAa,EAAE,uBAAuB,EAAE,mBAAmB,CAAC,CAAC;AACtG,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAqB,EACrB,IAA8E;IAE9E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACrC,KAAK,CAAC,sBAAsB,KAAK,CAAC,MAAM,iBAAiB,CAAC,CAAC;IAC3D,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;YAClD,mBAAmB,GAAG,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,YAAY,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;gBAClD,OAAO,CAAC,KAAK,CAAC,qDAAqD,OAAO,EAAE,CAAC,CAAC;gBAC9E,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,mBAAmB,EAAE,CAAC;YACtB,IAAI,mBAAmB,IAAI,iBAAiB,EAAE,CAAC;gBAC7C,OAAO,CAAC,KAAK,CAAC,mBAAmB,mBAAmB,uCAAuC,CAAC,CAAC;gBAC7F,MAAM,IAAI,KAAK,CAAC,kDAAkD,iBAAiB,GAAG,EAAE;oBACtF,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,0BAA0B,YAAY,YAAY,OAAO,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,aAAa,YAAY,uDAAuD,CAAC,CAAC;QAC/F,GAAG,CAAC,kBAAkB,YAAY,EAAE,CAAC,CAAC;IACxC,CAAC;AACH,CAAC"}

package/dist/ingest/points.js

@@ -0,0 +1,50 @@
+/**
+ * Chunks -> Qdrant points, shared by batch ingest (src/ingest.ts) and
+ * single-file ingest (src/tools/hydrate.ts) so both write the same ids and
+ * payloads.
+ */
+import { createHash } from "node:crypto";
+import { config } from "../config.js";
+import { embedBatch } from "../services/embeddings.js";
+/** The text a chunk is embedded and indexed as: its heading path, a blank line, its body. */
+export function chunkText(c) {
+    return `${c.section}\n\n${c.text}`;
+}
+/**
+ * Embed a doc's chunks and replace the doc's points in Qdrant. Point ids
+ * are derived from (source, section, index), so a section that was renamed
+ * or removed would otherwise stay in the collection for ever and keep being
+ * retrieved: the doc's old points are deleted before the upsert. A chunk
+ * whose embedding failed is left out. Without a BM25 encoder the sparse
+ * vector is empty (dense-only contribution). Returns the points written.
+ */
+export async function replaceDocPoints(qdrant, doc) {
+    const { source, chunks, bm25 } = doc;
+    const texts = chunks.map(chunkText);
+    const vectors = await embedBatch(texts);
+    const points = chunks.flatMap((c, i) => {
+        const dense = vectors[i];
+        if (!dense)
+            return [];
+        const text = chunkText(c);
+        return [
+            {
+                id: createHash("sha256").update(`${source}:${c.section}:${i}`).digest("hex").slice(0, 32),
+                dense,
+                sparse: bm25 ? bm25.encode(text) : { indices: [], values: [] },
+                payload: {
+                    source,
+                    section: c.section,
+                    text,
+                    embed_model: config.ollama.model,
+                    ingested_at: new Date().toISOString(),
+                },
+            },
+        ];
+    });
+    await qdrant.deleteBySource(source);
+    if (points.length > 0)
+        await qdrant.upsertChunks(points);
+    return points.length;
+}
+//# sourceMappingURL=points.js.map

package/dist/ingest/points.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"points.js","sourceRoot":"","sources":["../../src/ingest/points.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,6FAA6F;AAC7F,MAAM,UAAU,SAAS,CAAC,CAAW;IACnC,OAAO,GAAG,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AACrC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAqB,EACrB,GAAqE;IAErE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO;YACL;gBACE,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACzF,KAAK;gBACL,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;gBAC9D,OAAO,EAAE;oBACP,MAAM;oBACN,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,IAAI;oBACJ,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;oBAChC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC"}