lens-content-processor 0.79.0 → 0.80.0

package/dist/engine/protocol.d.ts

@@ -0,0 +1,146 @@
+import type { ContentError } from "../index.js";
+/**
+ * Which files of a checkout are content: each top-level directory that holds
+ * content → the endings its files' names may have, such as
+ * {"modules": [".md", ".timestamps.json"], "widgets": [".md", ".html"]}. The
+ * platform sends its own list (core/content/git_fetcher.content_directories),
+ * so the engine keeps none.
+ */
+export type ContentDirectories = Record<string, string[]>;
+/** Where a job's content comes from. */
+export type JobSource = 
+/** A checkout of the content repository; the job reads the content files (`directories`) of the tree of `commit`. */
+{
+    type: "git";
+    repoDir: string;
+    commit: string;
+    directories: ContentDirectories;
+}
+/** An explicit file map, as `cli.ts --stdin` takes it (path → content). */
+ | {
+    type: "files";
+    files: Record<string, string>;
+};
+export type JobKind = 
+/** Bring the build layer up to date with the source; store the results. */
+"commit"
+/** Like commit, but every unit runs again (a new validator version, or a damaged store). */
+ | "cold"
+/** Build the source with `overlays` applied, and store nothing. For agents' checks. */
+ | "overlay";
+export interface Job {
+    kind: JobKind;
+    source: JobSource;
+    /** The build-layer file, or ":memory:" for a store that lives only in this process. */
+    storePath: string;
+    /** Overlay jobs: text that replaces a file for this job only; null deletes the file. */
+    overlays?: Record<string, string | null>;
+    /** Units that killed an earlier attempt of this job: not run (worker.ts, blame and skip). */
+    skipUnits?: string[];
+    /**
+     * Run every unit that the build layer marks failed, not only those whose
+     * inputs changed: one that hung once, or was killed from outside, may pass
+     * now. A unit that fails again is blamed again. The platform sets it on the
+     * first job a process runs against a store.
+     */
+    retryFailed?: boolean;
+    /** Write today's complete CLI output (byte-identical to `cli.ts --stdin --defer-urls`) here. */
+    fullOutputPath?: string;
+    /**
+     * Leave the course-graph checks out of the full output's errors, as
+     * `cli.ts --skip-course-graph` does (the platform runs them as a job of
+     * their own and merges them itself).
+     */
+    skipCourseGraph?: boolean;
+    /**
+     * Overlay jobs: the paths whose issues are the overlay's own
+     * (OverlayReport.issues); by default the overlaid paths.
+     */
+    focusPaths?: string[];
+    /**
+     * Overlay jobs: the content paths built with their pending CriticMarkup
+     * suggestions accepted (acceptAllSuggestions, authoring-markup.ts), the
+     * drafts view. Every other file is built as published, its suggestions
+     * rejected. Absent or empty: the approved view.
+     */
+    draftPaths?: string[];
+}
+export type PageChange = {
+    sourcePath: string;
+    index: number;
+    slug: string;
+    change: "added" | "changed" | "removed";
+};
+/** A content file whose version differs from the one the build layer held before the job. */
+export type FileChange = {
+    path: string;
+    change: "added" | "changed" | "removed";
+};
+export interface JobResult {
+    event: "result";
+    status: "ok" | "error";
+    /** Set when status is "error". */
+    message?: string;
+    validatorVersion: string;
+    /** The commit the build layer now describes (git sources). */
+    commit?: string;
+    /**
+     * Advice ("What a reader applies" above): pages whose final JSON this job
+     * added, changed or removed in the build layer, in canonical order.
+     */
+    changedPages: PageChange[];
+    /** Advice: whole-content artifacts whose JSON this job changed ("errors", "courses", ...). */
+    changedArtifacts: string[];
+    /** Advice: content files whose version differs from the build layer's before the job, in path order. */
+    changedFiles?: FileChange[];
+    /** Path units (one per file) that ran, and units skipped because they are marked failed. */
+    unitsRun: number;
+    skippedUnits: string[];
+    /** Finish units (one per page) that ran. */
+    finishUnitsRun?: number;
+    /**
+     * The URL report of `cli.ts --defer-urls` for the job's content: issues
+     * from still-fresh entries of the URL cache file, and the number of URLs
+     * that need a live check (commit, cold and overlay jobs).
+     */
+    urlIssues?: ContentError[];
+    staleUrlCount?: number;
+    /** Milliseconds per stage, for the logs. */
+    timings: Record<string, number>;
+    /** Overlay jobs only: what the overlay would change. */
+    overlay?: OverlayReport;
+}
+export interface OverlayReport {
+    /** Issues whose file is one of focusPaths, as the content would be with the overlay. */
+    issues: ContentError[];
+    /** Issues in other files that the overlay adds (not present without it). */
+    causedIssues: ContentError[];
+    /** Pages whose final JSON the overlay would change. */
+    changedPages: PageChange[];
+    /** The course reachability map of the checked content (fileToCourses, as cli.ts emits it). */
+    fileToCourses: Record<string, string[]>;
+}
+/** A progress event (worker.ts): the unit the worker is about to run, or null once it has finished. */
+export interface UnitEvent {
+    event: "unit";
+    id: string | null;
+}
+/**
+ * The content of one job. `versions()` lists every content file; the version of a
+ * file is the Git blob id of its raw bytes. `read()` returns the text exactly as
+ * the platform passes it to the processor today (UTF-8, universal newlines; see
+ * core/content/git_fetcher.read_all_files).
+ */
+export interface ContentSource {
+    versions(): Map<string, string>;
+    read(path: string): string;
+}
+/**
+ * Calls the engine makes around each unit: the worker reports progress and
+ * records the running unit (worker.ts); tests see which units run, and stop a
+ * job part way.
+ */
+export interface JobHooks {
+    beforeUnit?(unitId: string): void;
+    afterUnit?(unitId: string): void;
+}

package/dist/engine/protocol.js

@@ -0,0 +1,40 @@
+// The job protocol between the platform and the content worker: the Job the
+// worker reads, the JobResult it writes last, and what the build layer holds
+// afterwards. How the worker runs, its events, and blame and skip: worker.ts.
+// Results that are large (pages, errors) are read from the build layer, not
+// from stdout.
+//
+// Artifacts (the build layer's artifacts table; JobResult.changedArtifacts
+// names the ones a job changed):
+//
+//   errors          every content error, in today's order, without the
+//                   course-graph checks (they run as their own job)
+//   errorSources    one entry per entry of errors: the ids of the units that
+//                   reported it, in the order they did. "path:<file>" is a
+//                   file's unit, "page:<file>#<index>" a page's finish step,
+//                   "whole" a check across files (no unit, no reads). A
+//                   unit's reads are in the reads table. A finish step's
+//                   "flat:<file>#<index>" read is the flattened page that unit
+//                   "path:<file>" made: follow that unit's reads to files.
+//   courses, surveys, meetingDocs, articleSegments, urlsToValidate, skillTree
+//                   as in processContent's result; meetingDocs and
+//                   articleSegments are absent when empty
+//   fileToCourses   the course reachability map, as cli.ts emits it
+//   dropped         what errors left out of pages, for the staging admins'
+//                   error boxes: page slug → {page, sourcePath, items,
+//                   moduleError} (src/dropped.ts). Not in the CLI output.
+//
+// What a reader applies. A job commits its writes in batches while it runs,
+// so a job that fails or is killed can leave pages and artifacts written that
+// no result ever listed, and the next job finds them up to date and does not
+// list them either. So a reader such as the platform's cache keeps the state
+// it applied last and, after every job whose status is "ok", reads the build
+// layer's state in one read transaction and applies what differs: the hash of
+// each page, the hash of each artifact (the meta keys "artifact:*"), the file
+// versions and the commit (schema: sqlite-store.ts). After a failed or killed
+// job that state can mix the old and the new content, so a reader applies
+// nothing then; the next ok job brings everything it missed. A JobResult's
+// changedPages, changedArtifacts and changedFiles are advice, for logs: what
+// the job changed relative to the build layer before it ran.
+export {};
+//# sourceMappingURL=protocol.js.map

package/dist/engine/protocol.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../../src/engine/protocol.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,6EAA6E;AAC7E,8EAA8E;AAC9E,4EAA4E;AAC5E,eAAe;AACf,EAAE;AACF,2EAA2E;AAC3E,iCAAiC;AACjC,EAAE;AACF,uEAAuE;AACvE,oEAAoE;AACpE,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,wEAAwE;AACxE,yEAAyE;AACzE,+EAA+E;AAC/E,2EAA2E;AAC3E,8EAA8E;AAC9E,mEAAmE;AACnE,0DAA0D;AAC1D,oEAAoE;AACpE,2EAA2E;AAC3E,uEAAuE;AACvE,0EAA0E;AAC1E,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAC9E,0EAA0E;AAC1E,2EAA2E;AAC3E,6EAA6E;AAC7E,6DAA6D"}

package/dist/engine/reader.d.ts

@@ -0,0 +1,133 @@
+import { type ContentTier } from "../validator/tier.js";
+import type { RecordedRead } from "./store.js";
+/** What a view tells its log. */
+export interface ReadObserver {
+    get(key: string): void;
+    has(key: string): void;
+    /** The caller looked at the whole set of keys (iteration or size). */
+    scan(): void;
+    /** The caller read only a file's link facts (readFacts). */
+    facts?(path: string): void;
+    /** The caller asked which paths look like a missing one: `<dir>|<path>`. */
+    similar?(query: string): void;
+    /** The caller iterated every value; without this hook, a scan and a get of each key. */
+    all?(): void;
+}
+/** A read-only map from path to file text, tier or tags, or from key to a table entry. */
+export declare class ContentView<V> implements ReadonlyMap<string, V> {
+    private readonly map;
+    private readonly observer?;
+    private findSimilar;
+    constructor(map: ReadonlyMap<string, V>, observer?: ReadObserver | undefined);
+    get(key: string): V | undefined;
+    has(key: string): boolean;
+    get size(): number;
+    keys(): MapIterator<string>;
+    values(): MapIterator<V>;
+    entries(): MapIterator<[string, V]>;
+    [Symbol.iterator](): MapIterator<[string, V]>;
+    forEach(callback: (value: V, key: string, map: ReadonlyMap<string, V>) => void, thisArg?: unknown): void;
+    /** A file's link facts, recorded as facts:<path> (a whole read without a facts observer). */
+    facts(this: ContentView<string>, path: string): LinkFacts | undefined;
+    /**
+     * The paths findSimilarFiles suggests for the missing `path`, recorded as
+     * similar:<dir>|<path> (a scan without a similar observer).
+     */
+    similar(this: ContentView<string>, path: string, dir: string): string[];
+    /** Iterating values reads every entry. */
+    private readAll;
+}
+export type ContentFiles = ContentView<string>;
+export type TierMap = ContentView<ContentTier>;
+export type TagMap = ContentView<string[]>;
+/**
+ * What a card or an inline link shows of the file it points to
+ * (flattener/resolve-text-links.ts): the raw frontmatter values, which the
+ * link turns into a title, a slug, a content id and a summary. Nothing else
+ * of the target's text reaches the linking page.
+ */
+export interface LinkFacts {
+    title?: unknown;
+    slug?: unknown;
+    id?: unknown;
+    tldr?: unknown;
+}
+export declare function linkFacts(path: string, text: string): LinkFacts;
+/**
+ * The link facts of a file in `files`, which must hold it. A view records the
+ * read as facts:<path>; a plain map (tests call rules with one) is read as is.
+ */
+export declare function readFacts(files: ContentFiles, path: string): LinkFacts;
+/** The version a facts:<path> read sees. Non-finite numbers are tagged: JSON writes them as null. */
+export declare function factsVersion(facts: LinkFacts): string;
+/** Path order: code-unit order, the order the processor visits files in. */
+export declare function byPath(a: string, b: string): number;
+/**
+ * `files` in path order, as a view that records nothing. Callers list files
+ * in whatever order they found them (a file system walk, a JSON object), and
+ * every output order is derived from this one, so the output depends on the
+ * content alone.
+ */
+export declare function inPathOrder(files: ReadonlyMap<string, string>): ContentFiles;
+/** The views one unit (or one whole-content step) reads content through. */
+export interface ContentViews {
+    files: ContentFiles;
+    tierMap: TierMap;
+    tagMap: TagMap;
+}
+/** Answers the current version of a read key; how a stored unit's reads are checked. */
+export interface Versions {
+    versionOf(key: string): string;
+}
+/**
+ * The content of one job: every file's version and text, and the tags of
+ * every Markdown and widget file. It answers the current version of any
+ * content read key (file, facts, exists, paths, similar, tier, tags).
+ */
+export declare class ContentState implements Versions {
+    readonly versions: ReadonlyMap<string, string>;
+    readonly texts: ReadonlyMap<string, string>;
+    readonly tags: ReadonlyMap<string, string[]>;
+    readonly tiers: Map<string, ContentTier>;
+    private pathsHash;
+    private readonly factsVersions;
+    private readonly similarVersions;
+    private findSimilar;
+    /**
+     * @param versions path → Git blob id, for every file
+     * @param texts path → text, with keys in sorted order (the order units see)
+     * @param tags lower-cased frontmatter tags of every Markdown and widget file
+     * @param factsVersions path → factsVersion of its link facts, when known
+     *   (the engine keeps them per file); others are computed from the text
+     */
+    constructor(versions: ReadonlyMap<string, string>, texts: ReadonlyMap<string, string>, tags: ReadonlyMap<string, string[]>, factsVersions?: ReadonlyMap<string, string>);
+    /** The version a read of `key` sees in this content. */
+    versionOf(key: string): string;
+    /**
+     * A hash of what findSimilarFiles suggests for `query` (`<dir>|<path>`) in
+     * this content: a function of the query and the path list alone, so after
+     * a file is added or removed it tells which suggestions changed.
+     */
+    private similarVersion;
+    /** Views that record nothing. */
+    views(): ContentViews;
+    /** Views that write every lookup to `log`. */
+    recordingViews(log: ReadLog): ContentViews;
+}
+/**
+ * A view of a whole-content table whose lookups `log` records as
+ * `<prefix><key>` (any lookup of a key, found or not, is one read of it).
+ */
+export declare function tableView<V>(table: ReadonlyMap<string, V>, prefix: string, log?: ReadLog): ContentView<V>;
+/** The version of a table entry: a hash of its JSON, or "-" when there is none. */
+export declare function entryVersion(value: unknown): string;
+/** The reads of one unit, with the version each saw. */
+export declare class ReadLog {
+    private readonly versions;
+    private readonly seen;
+    constructor(versions: Versions);
+    read(key: string): void;
+    /** Every read, sorted by key. */
+    reads(): RecordedRead[];
+}
+export declare function sha1(text: string): string;

package/dist/engine/reader.js

@@ -0,0 +1,298 @@
+// The reader: the only way a unit sees content, and the record of what it saw.
+//
+// A unit (one content file's pass, runPathUnit in ../index.ts, or one page's
+// finish step, finishPage) reads other files' text, their facts, tiers and
+// tags, and the whole-content step's tables, through ContentViews. A view
+// built with a ReadLog writes down every lookup as a key and the version it
+// saw:
+//
+//   file:<path>      get(path) on the files     the Git blob id, or "missing"
+//   facts:<path>     readFacts(files, path)     a hash of the link facts, or "missing"
+//   exists:<path>    has(path) on any view      "1" or "0"
+//   paths:*          keys(), size, iteration    a hash of the sorted path list
+//   similar:<q>      similar(path, dir) on the  a hash of the paths findSimilarFiles
+//                    files; q is <dir>|<path>   suggests for the missing path
+//   similar:*        with every similar: read   "1": finds the units to check
+//                                               when a file is added or removed
+//   tier:<path>      get(path) on the tiers     the tier, or "-"
+//   tags:<path>      get(path) on the tags      the tags as JSON, or "-"
+//   table:<t>:<key>  get(key) on a table view   a hash of the entry, or "-"
+//   glossary:<path>  get(path) on the glossary  a hash of the entry, or "-"
+//   flat:<page>      a finish unit's input      a hash of the flattened page
+//
+// After an edit, a unit runs again only when a key it read has a new version
+// (build.ts). A unit that only checked that a file exists does not run again
+// when that file's text changes; a card or link that shows a lens's title
+// does not run again when the lens's body changes; a tag edit that leaves
+// the tier alone does not rerun the units that read only the tier; and a
+// unit with a broken link runs again when a file is added or removed only if
+// its "did you mean" suggestion changes.
+//
+// The views are nominal: a Map is not a ContentView, so TypeScript refuses a
+// raw map where a unit or a rule takes one. That is what keeps every read on
+// the record.
+//
+// Two more rules keep an incremental build equal to a cold one. Break either
+// and a page stays stale after an edit, with no test failing at once:
+// - A memo shared between units is safe only when it is keyed by a text the
+//   unit read through a view (runMemo in ../markdown-parse.ts): it gives only
+//   what that text gives. The --cache-dir caches return results without the
+//   reads that made them, so only processContent uses them (runPathUnit's
+//   steps).
+// - A unit's output is plain data, with no Map or class instance: the build
+//   layer stores it as JSON.
+import { createHash } from "crypto";
+import { parseFrontmatter } from "../parser/frontmatter.js";
+import { similarFilesIn } from "../parser/wikilink.js";
+import { buildTierMap } from "../validator/tier.js";
+/** A read-only map from path to file text, tier or tags, or from key to a table entry. */
+export class ContentView {
+    map;
+    observer;
+    findSimilar;
+    constructor(map, observer) {
+        this.map = map;
+        this.observer = observer;
+    }
+    get(key) {
+        this.observer?.get(key);
+        return this.map.get(key);
+    }
+    has(key) {
+        this.observer?.has(key);
+        return this.map.has(key);
+    }
+    get size() {
+        this.observer?.scan();
+        return this.map.size;
+    }
+    keys() {
+        this.observer?.scan();
+        return this.map.keys();
+    }
+    values() {
+        this.readAll();
+        return this.map.values();
+    }
+    entries() {
+        this.readAll();
+        return this.map.entries();
+    }
+    [Symbol.iterator]() {
+        return this.entries();
+    }
+    forEach(callback, thisArg) {
+        this.readAll();
+        this.map.forEach((value, key) => callback.call(thisArg, value, key, this));
+    }
+    /** A file's link facts, recorded as facts:<path> (a whole read without a facts observer). */
+    facts(path) {
+        if (this.observer?.facts)
+            this.observer.facts(path);
+        else
+            this.observer?.get(path);
+        const text = this.map.get(path);
+        return text === undefined ? undefined : linkFacts(path, text);
+    }
+    /**
+     * The paths findSimilarFiles suggests for the missing `path`, recorded as
+     * similar:<dir>|<path> (a scan without a similar observer).
+     */
+    similar(path, dir) {
+        if (this.observer?.similar)
+            this.observer.similar(`${dir}|${path}`);
+        else
+            this.observer?.scan();
+        this.findSimilar ??= similarFilesIn(this.map.keys());
+        return this.findSimilar(path, dir);
+    }
+    /** Iterating values reads every entry. */
+    readAll() {
+        if (!this.observer)
+            return;
+        if (this.observer.all)
+            return this.observer.all();
+        this.observer.scan();
+        for (const key of this.map.keys())
+            this.observer.get(key);
+    }
+}
+export function linkFacts(path, text) {
+    const { frontmatter } = parseFrontmatter(text, path);
+    const { title, slug, id, tldr } = frontmatter;
+    return { title, slug, id, tldr };
+}
+/**
+ * The link facts of a file in `files`, which must hold it. A view records the
+ * read as facts:<path>; a plain map (tests call rules with one) is read as is.
+ */
+export function readFacts(files, path) {
+    if (files instanceof ContentView)
+        return files.facts(path);
+    return linkFacts(path, files.get(path));
+}
+/** The version a facts:<path> read sees. Non-finite numbers are tagged: JSON writes them as null. */
+export function factsVersion(facts) {
+    return sha1(JSON.stringify(facts, (_key, value) => typeof value === "number" && !Number.isFinite(value)
+        ? `\u0000${value}`
+        : value));
+}
+/** Path order: code-unit order, the order the processor visits files in. */
+export function byPath(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+/**
+ * `files` in path order, as a view that records nothing. Callers list files
+ * in whatever order they found them (a file system walk, a JSON object), and
+ * every output order is derived from this one, so the output depends on the
+ * content alone.
+ */
+export function inPathOrder(files) {
+    return new ContentView(new Map([...files].sort(([a], [b]) => byPath(a, b))));
+}
+/**
+ * The content of one job: every file's version and text, and the tags of
+ * every Markdown and widget file. It answers the current version of any
+ * content read key (file, facts, exists, paths, similar, tier, tags).
+ */
+export class ContentState {
+    versions;
+    texts;
+    tags;
+    tiers;
+    pathsHash;
+    factsVersions;
+    similarVersions = new Map();
+    findSimilar;
+    /**
+     * @param versions path → Git blob id, for every file
+     * @param texts path → text, with keys in sorted order (the order units see)
+     * @param tags lower-cased frontmatter tags of every Markdown and widget file
+     * @param factsVersions path → factsVersion of its link facts, when known
+     *   (the engine keeps them per file); others are computed from the text
+     */
+    constructor(versions, texts, tags, factsVersions = new Map()) {
+        this.versions = versions;
+        this.texts = texts;
+        this.tags = tags;
+        this.tiers = buildTierMap(tags);
+        this.factsVersions = new Map(factsVersions);
+    }
+    /** The version a read of `key` sees in this content. */
+    versionOf(key) {
+        const colon = key.indexOf(":");
+        const kind = key.slice(0, colon);
+        const path = key.slice(colon + 1);
+        switch (kind) {
+            case "file":
+                return this.versions.get(path) ?? "missing";
+            case "facts": {
+                if (!this.versions.has(path))
+                    return "missing";
+                let version = this.factsVersions.get(path);
+                if (version === undefined) {
+                    version = factsVersion(linkFacts(path, this.texts.get(path)));
+                    this.factsVersions.set(path, version);
+                }
+                return version;
+            }
+            case "exists":
+                return this.versions.has(path) ? "1" : "0";
+            case "paths":
+                return (this.pathsHash ??= hashPaths(this.versions.keys()));
+            case "similar":
+                return path === "*" ? "1" : this.similarVersion(path);
+            case "tier":
+                return this.tiers.get(path) ?? "-";
+            case "tags": {
+                const tags = this.tags.get(path);
+                return tags ? JSON.stringify(tags) : "-";
+            }
+            default:
+                throw new Error(`Unknown read key: ${key}`);
+        }
+    }
+    /**
+     * A hash of what findSimilarFiles suggests for `query` (`<dir>|<path>`) in
+     * this content: a function of the query and the path list alone, so after
+     * a file is added or removed it tells which suggestions changed.
+     */
+    similarVersion(query) {
+        let version = this.similarVersions.get(query);
+        if (version === undefined) {
+            const bar = query.indexOf("|");
+            this.findSimilar ??= similarFilesIn(this.texts.keys());
+            const found = this.findSimilar(query.slice(bar + 1), query.slice(0, bar));
+            version = entryVersion(found);
+            this.similarVersions.set(query, version);
+        }
+        return version;
+    }
+    /** Views that record nothing. */
+    views() {
+        return {
+            files: new ContentView(this.texts),
+            tierMap: new ContentView(this.tiers),
+            tagMap: new ContentView(this.tags),
+        };
+    }
+    /** Views that write every lookup to `log`. */
+    recordingViews(log) {
+        const observer = (kind) => ({
+            get: (path) => log.read(`${kind}:${path}`),
+            has: (path) => log.read(`exists:${path}`),
+            scan: () => log.read("paths:*"),
+            facts: (path) => log.read(`facts:${path}`),
+            similar: (query) => {
+                log.read("similar:*");
+                log.read(`similar:${query}`);
+            },
+        });
+        return {
+            files: new ContentView(this.texts, observer("file")),
+            tierMap: new ContentView(this.tiers, observer("tier")),
+            tagMap: new ContentView(this.tags, observer("tags")),
+        };
+    }
+}
+/**
+ * A view of a whole-content table whose lookups `log` records as
+ * `<prefix><key>` (any lookup of a key, found or not, is one read of it).
+ */
+export function tableView(table, prefix, log) {
+    return new ContentView(table, log && {
+        get: (key) => log.read(prefix + key),
+        has: (key) => log.read(prefix + key),
+        scan: () => log.read(`${prefix}*`),
+    });
+}
+/** The version of a table entry: a hash of its JSON, or "-" when there is none. */
+export function entryVersion(value) {
+    return value === undefined ? "-" : sha1(JSON.stringify(value));
+}
+/** The reads of one unit, with the version each saw. */
+export class ReadLog {
+    versions;
+    seen = new Map();
+    constructor(versions) {
+        this.versions = versions;
+    }
+    read(key) {
+        if (!this.seen.has(key))
+            this.seen.set(key, this.versions.versionOf(key));
+    }
+    /** Every read, sorted by key. */
+    reads() {
+        return [...this.seen]
+            .sort(([a], [b]) => byPath(a, b))
+            .map(([key, version]) => ({ key, version }));
+    }
+}
+export function sha1(text) {
+    return createHash("sha1").update(text).digest("hex");
+}
+function hashPaths(paths) {
+    const sorted = [...paths].sort(byPath);
+    return sha1(sorted.join("\n"));
+}
+//# sourceMappingURL=reader.js.map

package/dist/engine/reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../src/engine/reader.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,0EAA0E;AAC1E,4EAA4E;AAC5E,OAAO;AACP,EAAE;AACF,8EAA8E;AAC9E,uFAAuF;AACvF,2DAA2D;AAC3D,+EAA+E;AAC/E,qFAAqF;AACrF,8EAA8E;AAC9E,8EAA8E;AAC9E,gFAAgF;AAChF,iEAAiE;AACjE,yEAAyE;AACzE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,0EAA0E;AAC1E,0EAA0E;AAC1E,yEAAyE;AACzE,6EAA6E;AAC7E,yCAAyC;AACzC,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,cAAc;AACd,EAAE;AACF,6EAA6E;AAC7E,sEAAsE;AACtE,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,0EAA0E;AAC1E,YAAY;AACZ,4EAA4E;AAC5E,6BAA6B;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAoB,MAAM,sBAAsB,CAAC;AAmBtE,0FAA0F;AAC1F,MAAM,OAAO,WAAW;IAIH;IACA;IAJX,WAAW,CAA2B;IAE9C,YACmB,GAA2B,EAC3B,QAAuB;QADvB,QAAG,GAAH,GAAG,CAAwB;QAC3B,aAAQ,GAAR,QAAQ,CAAe;IACvC,CAAC;IAEJ,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,IAAI;QACN,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,IAAI;QACF,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,CACL,QAAsE,EACtE,OAAiB;QAEjB,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,6FAA6F;IAC7F,KAAK,CAA4B,IAAY;QAC3C,IAAI,IAAI,CAAC,QAAQ,EAAE,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;;YAC/C,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;IAED;;;OAGG;IACH,OAAO,CAA4B,IAAY,EAAE,GAAW;QAC1D,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO;YAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,CAAC;;YAC/D,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,KAAK,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,0CAA0C;IAClC,OAAO;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACrB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;CACF;AAmBD,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,IAAY;IAClD,MAAM,EAAE,WAAW,EAAE,GAAG,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC;IAC9C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,KAAmB,EAAE,IAAY;IACzD,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;IAC5D,OAAO,SAAS,CAAC,IAAI,EAAG,KAAqC,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC;AAC5E,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,YAAY,CAAC,KAAgB;IAC3C,OAAO,IAAI,CACT,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACpC,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAClD,CAAC,CAAC,SAAS,KAAK,EAAE;QAClB,CAAC,CAAC,KAAK,CACV,CACF,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,MAAM,CAAC,CAAS,EAAE,CAAS;IACzC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,KAAkC;IAC5D,OAAO,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/E,CAAC;AAcD;;;;GAIG;AACH,MAAM,OAAO,YAAY;IAeZ;IACA;IACA;IAhBF,KAAK,CAA2B;IACjC,SAAS,CAAqB;IACrB,aAAa,CAAsB;IACnC,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrD,WAAW,CAA2B;IAE9C;;;;;;OAMG;IACH,YACW,QAAqC,EACrC,KAAkC,EAClC,IAAmC,EAC5C,gBAA6C,IAAI,GAAG,EAAE;QAH7C,aAAQ,GAAR,QAAQ,CAA6B;QACrC,UAAK,GAAL,KAAK,CAA6B;QAClC,SAAI,GAAJ,IAAI,CAA+B;QAG5C,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED,wDAAwD;IACxD,SAAS,CAAC,GAAW;QACnB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC;YAC9C,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,OAAO,SAAS,CAAC;gBAC/C,IAAI,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC,CAAC;oBAC/D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBACxC,CAAC;gBACD,OAAO,OAAO,CAAC;YACjB,CAAC;YACD,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAC7C,KAAK,OAAO;gBACV,OAAO,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC9D,KAAK,SAAS;gBACZ,OAAO,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YACxD,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YACrC,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACjC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YAC3C,CAAC;YACD;gBACE,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,cAAc,CAAC,KAAa;QAClC,IAAI,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,WAAW,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC1E,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,iCAAiC;IACjC,KAAK;QACH,OAAO;YACL,KAAK,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAClC,OAAO,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YACpC,MAAM,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;SACnC,CAAC;IACJ,CAAC;IAED,8CAA8C;IAC9C,cAAc,CAAC,GAAY;QACzB,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC;YAChD,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;YAC1C,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YACzC,IAAI,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YAC/B,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjB,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;YAC/B,CAAC;SACF,CAAC,CAAC;QACH,OAAO;YACL,KAAK,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YACpD,OAAO,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YACtD,MAAM,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;SACrD,CAAC;IACJ,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CACvB,KAA6B,EAC7B,MAAc,EACd,GAAa;IAEb,OAAO,IAAI,WAAW,CACpB,KAAK,EACL,GAAG,IAAI;QACL,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;QACpC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;QACpC,IAAI,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC;KACnC,CACF,CAAC;AACJ,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,wDAAwD;AACxD,MAAM,OAAO,OAAO;IAGW;IAFZ,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IAElD,YAA6B,QAAkB;QAAlB,aAAQ,GAAR,QAAQ,CAAU;IAAG,CAAC;IAEnD,IAAI,CAAC,GAAW;QACd,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,iCAAiC;IACjC,KAAK;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;aAClB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC;CACF;AAED,MAAM,UAAU,IAAI,CAAC,IAAY;IAC/B,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,SAAS,CAAC,KAAuB;IACxC,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACjC,CAAC"}