lens-content-processor 0.78.0 → 0.80.0

package/dist/engine/sources.d.ts

@@ -0,0 +1,39 @@
+import type { ContentSource } from "./protocol.js";
+/** The Git blob id of a file with this text: what `git hash-object` prints. */
+export declare function blobId(text: string): string;
+/** A source over a file map, as `cli.ts --stdin` takes it (path → text). */
+export declare function mapSource(files: Record<string, string> | Map<string, string>): ContentSource;
+/**
+ * `source` with `overlays` applied: a text replaces a file or adds one, null
+ * deletes one. Overlaid files keep their place in the order; new ones go last.
+ */
+export declare function overlaySource(source: ContentSource, overlays: Record<string, string | null>): ContentSource;
+/**
+ * `source` with each of `paths` that has pending CriticMarkup suggestions in
+ * its drafts view (every suggestion accepted), with that text's blob id as its
+ * version, so a job rebuilds it and everything that reads it.
+ */
+export declare function draftsSource(source: ContentSource, paths: Iterable<string>): ContentSource;
+/**
+ * The texts of a source as a map whose keys come in `order`. Texts are read
+ * on first use and kept, so a job that touches a few files reads only those.
+ * Units read texts here, so a source that fails (for example `git cat-file`
+ * dies) throws a Showstopper: the job fails, and the unit's file is not blamed.
+ */
+export declare class SourceTexts implements ReadonlyMap<string, string> {
+    private readonly source;
+    private readonly index;
+    private readonly memo;
+    private everything;
+    constructor(source: ContentSource, order: Iterable<string>);
+    get(path: string): string | undefined;
+    has(path: string): boolean;
+    get size(): number;
+    keys(): MapIterator<string>;
+    values(): MapIterator<string>;
+    entries(): MapIterator<[string, string]>;
+    [Symbol.iterator](): MapIterator<[string, string]>;
+    forEach(callback: (value: string, key: string, map: ReadonlyMap<string, string>) => void, thisArg?: unknown): void;
+    /** Every text, in order (iterating the values reads every file). */
+    private all;
+}

package/dist/engine/sources.js

@@ -0,0 +1,137 @@
+// Content sources for the engine: an explicit file map, overlays and the
+// drafts view over any source, and a lazy text map over a source. The
+// Git-checkout source is in git-source.ts.
+import { createHash } from "crypto";
+import { acceptAllSuggestions } from "../authoring-markup.js";
+import { isContentTextFile } from "../parser/widget.js";
+import { Showstopper } from "./units.js";
+/** The Git blob id of a file with this text: what `git hash-object` prints. */
+export function blobId(text) {
+    const bytes = Buffer.from(text, "utf8");
+    return createHash("sha1")
+        .update(`blob ${bytes.length}\0`)
+        .update(bytes)
+        .digest("hex");
+}
+/** A source over a file map, as `cli.ts --stdin` takes it (path → text). */
+export function mapSource(files) {
+    const texts = files instanceof Map ? new Map(files) : new Map(Object.entries(files));
+    const versions = new Map();
+    for (const [path, text] of texts)
+        versions.set(path, blobId(text));
+    return {
+        versions: () => versions,
+        read(path) {
+            const text = texts.get(path);
+            if (text === undefined)
+                throw new Error(`No such file: ${path}`);
+            return text;
+        },
+    };
+}
+/**
+ * `source` with `overlays` applied: a text replaces a file or adds one, null
+ * deletes one. Overlaid files keep their place in the order; new ones go last.
+ */
+export function overlaySource(source, overlays) {
+    const versions = new Map(source.versions());
+    const texts = new Map();
+    for (const [path, text] of Object.entries(overlays)) {
+        if (text === null) {
+            versions.delete(path);
+        }
+        else {
+            versions.set(path, blobId(text));
+            texts.set(path, text);
+        }
+    }
+    return {
+        versions: () => versions,
+        read(path) {
+            if (!versions.has(path))
+                throw new Error(`No such file: ${path}`);
+            return texts.get(path) ?? source.read(path);
+        },
+    };
+}
+/**
+ * `source` with each of `paths` that has pending CriticMarkup suggestions in
+ * its drafts view (every suggestion accepted), with that text's blob id as its
+ * version, so a job rebuilds it and everything that reads it.
+ */
+export function draftsSource(source, paths) {
+    const versions = new Map(source.versions());
+    const drafts = new Map();
+    for (const path of new Set(paths)) {
+        if (!versions.has(path) || !isContentTextFile(path))
+            continue;
+        const text = source.read(path);
+        const accepted = acceptAllSuggestions(text);
+        if (accepted === text)
+            continue;
+        drafts.set(path, accepted);
+        versions.set(path, blobId(accepted));
+    }
+    return {
+        versions: () => versions,
+        read: (path) => drafts.get(path) ?? source.read(path),
+    };
+}
+/**
+ * The texts of a source as a map whose keys come in `order`. Texts are read
+ * on first use and kept, so a job that touches a few files reads only those.
+ * Units read texts here, so a source that fails (for example `git cat-file`
+ * dies) throws a Showstopper: the job fails, and the unit's file is not blamed.
+ */
+export class SourceTexts {
+    source;
+    index;
+    memo = new Map();
+    everything;
+    constructor(source, order) {
+        this.source = source;
+        this.index = new Map([...order].map((path) => [path, true]));
+    }
+    get(path) {
+        if (!this.index.has(path))
+            return undefined;
+        let text = this.memo.get(path);
+        if (text === undefined) {
+            try {
+                text = this.source.read(path);
+            }
+            catch (error) {
+                throw new Showstopper(`The content source could not read ${path}: ${error instanceof Error ? error.message : error}`, { cause: error });
+            }
+            this.memo.set(path, text);
+        }
+        return text;
+    }
+    has(path) {
+        return this.index.has(path);
+    }
+    get size() {
+        return this.index.size;
+    }
+    keys() {
+        return this.index.keys();
+    }
+    values() {
+        return this.all().values();
+    }
+    entries() {
+        return this.all().entries();
+    }
+    [Symbol.iterator]() {
+        return this.entries();
+    }
+    forEach(callback, thisArg) {
+        this.all().forEach((text, path) => callback.call(thisArg, text, path, this));
+    }
+    /** Every text, in order (iterating the values reads every file). */
+    all() {
+        this.everything ??= new Map([...this.index.keys()].map((path) => [path, this.get(path)]));
+        return this.everything;
+    }
+}
+//# sourceMappingURL=sources.js.map

package/dist/engine/sources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sources.js","sourceRoot":"","sources":["../../src/engine/sources.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,sEAAsE;AACtE,2CAA2C;AAE3C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,+EAA+E;AAC/E,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,UAAU,CAAC,MAAM,CAAC;SACtB,MAAM,CAAC,QAAQ,KAAK,CAAC,MAAM,IAAI,CAAC;SAChC,MAAM,CAAC,KAAK,CAAC;SACb,MAAM,CAAC,KAAK,CAAC,CAAC;AACnB,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,SAAS,CACvB,KAAmD;IAEnD,MAAM,KAAK,GACT,KAAK,YAAY,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK;QAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ;QACxB,IAAI,CAAC,IAAI;YACP,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAqB,EACrB,QAAuC;IAEvC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACjC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ;QACxB,IAAI,CAAC,IAAI;YACP,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;YAClE,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,MAAqB,EACrB,KAAuB;IAEvB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,QAAQ,KAAK,IAAI;YAAE,SAAS;QAChC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ;QACxB,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;KACtD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IAMH;IALF,KAAK,CAAoB;IACzB,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,UAAU,CAAkC;IAEpD,YACmB,MAAqB,EACtC,KAAuB;QADN,WAAM,GAAN,MAAM,CAAe;QAGtC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,GAAG,CAAC,IAAY;QACd,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QAC5C,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,WAAW,CACnB,qCAAqC,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,EAC9F,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,GAAG,CAAC,IAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,CACL,QAIS,EACT,OAAiB;QAEjB,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAChC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CACzC,CAAC;IACJ,CAAC;IAED,oEAAoE;IAC5D,GAAG;QACT,IAAI,CAAC,UAAU,KAAK,IAAI,GAAG,CACzB,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC,CAC9D,CAAC;QACF,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF"}

package/dist/engine/sqlite-store.d.ts

@@ -0,0 +1,99 @@
+import { type BuildStore, type PageRecord, type PageSummary, type UnitRecord } from "./store.js";
+/** Commit a batch after this many writes... */
+export declare const BATCH_WRITES = 1000;
+/** ...or once it is this old. A killed job loses at most this much finished work. */
+export declare const BATCH_MS = 1000;
+/** The part of the driver API the store uses (better-sqlite3; node:sqlite has the same names). */
+interface Statement {
+    run(...params: unknown[]): unknown;
+    get(...params: unknown[]): unknown;
+    all(...params: unknown[]): unknown[];
+}
+interface Database {
+    prepare(sql: string): Statement;
+    exec(sql: string): void;
+    close(): void;
+    readonly inTransaction: boolean;
+}
+/** The store file for a validator version: `<dir>/build-<version>.sqlite`. */
+export declare function storePathFor(dir: string, version: string): string;
+/** Delete a store file and the files SQLite and the running record keep beside it. */
+export declare function removeStoreFiles(path: string): void;
+/**
+ * Delete every other `build-*.sqlite` store in the directory of `path`, once the
+ * store at `path` is complete. Returns the names of the stores removed. Does
+ * nothing when `path` itself is not named `build-*.sqlite`.
+ */
+export declare function removeOtherStores(path: string): string[];
+/**
+ * SQLite's answer when a file is not a database or its structure is broken,
+ * also when it is the cause of the error the store threw in its place.
+ */
+export declare function isDamagedStoreError(error: unknown): boolean;
+export interface OpenOptions {
+    /** The validator version. A store written by another version counts as foreign. */
+    version?: string;
+    /**
+     * Open for reading only, inside one read transaction, so every read sees the
+     * same committed state however long the job runs (overlay jobs). The file must
+     * exist, and a damaged or foreign file is an error instead of a fresh start.
+     */
+    readonly?: boolean;
+}
+/**
+ * Open the store at `path`. A missing, damaged or foreign file (another schema or
+ * validator version) is replaced by an empty store, whose `fresh` is true: the next
+ * job builds everything. With `readonly`, such a file is an error instead.
+ */
+export declare function openSqliteStore(path: string, options?: OpenOptions): SqliteStore;
+export declare class SqliteStore implements BuildStore {
+    private readonly path;
+    private readonly readonly;
+    /** True when openSqliteStore started an empty store: the file was missing, damaged or foreign. */
+    readonly fresh: boolean;
+    private readonly db;
+    private readonly q;
+    private pending;
+    private batchStarted;
+    /** Depth of atomically() calls: no batch commits while it is above 0. */
+    private holding;
+    private runningFd;
+    private closed;
+    /** The error of a write that failed; the store refuses to write after one. */
+    private failure;
+    /** Use openSqliteStore. */
+    constructor(db: Database, path: string, fresh: boolean, readonly: boolean);
+    getMeta(key: string): string | undefined;
+    setMeta(key: string, value: string): void;
+    getFileVersions(): Map<string, string>;
+    setFileVersion(path: string, version: string | null): void;
+    getUnit(id: string): UnitRecord | undefined;
+    listUnitIds(prefix?: string): string[];
+    putUnit(record: UnitRecord): void;
+    deleteUnit(id: string): void;
+    unitsReading(keys: Iterable<string>): string[];
+    listPages(): PageSummary[];
+    getPage(sourcePath: string, index: number): PageRecord | undefined;
+    putPage(page: PageRecord): void;
+    deletePage(sourcePath: string, index: number): void;
+    getArtifact(name: string): string | undefined;
+    getArtifactHash(name: string): string;
+    putArtifact(name: string, json: string): void;
+    deleteArtifact(name: string): void;
+    setRunning(unitId: string | null): void;
+    getRunning(): string | null;
+    getFailedUnits(): Map<string, string>;
+    setFailedUnit(unitId: string, reason: string | null): void;
+    commit(): void;
+    atomically<T>(apply: () => T): T;
+    close(): void;
+    /** Run one write inside the current batch, opening one if needed; commit when the batch is due. */
+    private write;
+    /**
+     * The one batch rule: commit after BATCH_WRITES writes or once the batch is
+     * BATCH_MS old, but never inside atomically(), and never after a failed write.
+     */
+    private commitIfDue;
+    private refuseAfterFailure;
+}
+export {};