@opendatalabs/personal-server-ts-server 1.34.1 → 1.35.0

package/dist/storage/pdpp-binary-chunks.d.ts

@@ -0,0 +1,120 @@
+import type { Database } from "better-sqlite3";
+export declare const BINARY_CHUNK_BYTES: number;
+/** Legacy whole-row reads: windows are at least this big and at most this many. */
+export declare const LEGACY_MIN_WINDOW_BYTES: number;
+export declare const LEGACY_WINDOWS = 16;
+export declare const CANONICAL_INLINE_BYTES: number;
+export declare const BINARY_CHUNKS_MIGRATION = "\nCREATE TABLE IF NOT EXISTS pdpp_binary_manifests (\n namespace TEXT NOT NULL, blob_id TEXT NOT NULL, size_bytes INTEGER NOT NULL,\n PRIMARY KEY(namespace,blob_id));\nCREATE TABLE IF NOT EXISTS pdpp_binary_chunks (\n namespace TEXT NOT NULL, blob_id TEXT NOT NULL, position INTEGER NOT NULL, bytes BLOB NOT NULL,\n PRIMARY KEY(namespace,blob_id,position));\n";
+export type BinaryNamespace = "canonical" | "pinned" | "received" | "staging" | "payload" | "pending";
+/**
+ * What each namespace's bytes are for. A "reader" namespace holds bytes a
+ * reader can be served. Each one MUST have manifest insert and delete
+ * triggers in the visibility view (pdpp-replica-visibility-view.ts), or
+ * records stay stale when its chunked bytes arrive or leave. Add a new
+ * namespace here first; pdpp-replica-invariants.test.ts guards the triggers.
+ */
+export declare const BINARY_NAMESPACE_ROLE: {
+    readonly canonical: "reader";
+    readonly received: "reader";
+    readonly pending: "reader";
+    readonly pinned: "export";
+    readonly staging: "scratch";
+    readonly payload: "scratch";
+};
+/** Read chunk copies when present; fall back to main's whole-value blob row. */
+export declare function binaryLength(db: Database, namespace: BinaryNamespace, id: string): number | undefined;
+export declare function readBinaryChunks(db: Database, namespace: BinaryNamespace, id: string): Generator<Uint8Array>;
+/** Where a chunked read resumes: a chunk row and a byte offset inside it. */
+export interface BinaryCursor {
+    position: number;
+    offset: number;
+}
+/**
+ * Read exactly `length` bytes of chunked (manifest) bytes from `from`, and
+ * return them with the cursor after them. Rows may have any length, so a
+ * caller that reads a value in parts keeps the returned cursor; it never
+ * seeks by arithmetic. Memory is `length`. Legacy whole-row values are not
+ * supported: only namespaces without a legacy table use this.
+ */
+export declare function readBinaryWindow(db: Database, namespace: BinaryNamespace, id: string, from: BinaryCursor, length: number): {
+    bytes: Uint8Array;
+    next: BinaryCursor;
+};
+export declare function verifyBinary(db: Database, namespace: BinaryNamespace, id: string, digest: string, size: number): boolean;
+export declare function deleteBinary(db: Database, namespace: BinaryNamespace, id: string): void;
+/**
+ * A repair write is about to give `id` a verified copy in `namespace`.
+ * Remove a published manifest whose size is not a safe non-negative
+ * integer, with its chunks, in the caller's transaction. Readers veto a
+ * namespace on such a manifest even beside a legacy row, so without this a
+ * repair would never take effect.
+ */
+export declare function clearMalformedManifest(db: Database, namespace: BinaryNamespace, id: string): void;
+/** Publish the manifest only after EOF. Callers own transaction/visibility and cleanup. */
+export declare function binaryWriter(db: Database, namespace: BinaryNamespace, id: string): {
+    write(bytes: Uint8Array): void;
+    finish(): number;
+};
+export declare function copyBinary(db: Database, from: BinaryNamespace, sourceId: string, to: BinaryNamespace, targetId?: string): void;
+/**
+ * Write canonical bytes the server holds as one value, in one form only. A
+ * value up to CANONICAL_INLINE_BYTES (one Lite fragment) is main's whole row.
+ * A larger one is chunks only: the transport and capture's pin read those
+ * chunks, and the flag-off boot that retires replication writes main's row
+ * (`materializeCanonicalBinaries`).
+ */
+export declare function writeCanonicalBinary(db: Database, id: string, bytes: Uint8Array): void;
+/**
+ * Promote verified bytes to canonical in the caller's transaction. A value up
+ * to CANONICAL_INLINE_BYTES (one Lite fragment) becomes main's whole row. A
+ * larger one is copied chunk by chunk and is stored only in the chunk form:
+ * no whole value is assembled or stored twice. Canonical readers accept both
+ * forms; the flag-off boot that retires replication writes main's row for
+ * chunk-only blobs (`materializeCanonicalBinaries`).
+ */
+export declare function copyBinaryToCanonical(db: Database, from: BinaryNamespace, sourceId: string, targetId?: string): void;
+/**
+ * Write main's whole row for each verified canonical blob that has only the
+ * chunk form, so a flag-off or plain main build can read it. One blob per
+ * transaction; each costs one whole value in memory. Run it only when
+ * replication is retired: an enabled store keeps large blobs chunk-only.
+ * A first enable that stopped after the door committed, before the replica
+ * schema, left no chunk store, so there is nothing to write.
+ */
+export declare function materializeCanonicalBinaries(db: Database): number;
+/**
+ * Explicit whole-value read for HTTP consumers, rollback materialization and
+ * inline-size promotion. Replication never calls it on a value larger than
+ * CANONICAL_INLINE_BYTES.
+ */
+export declare function readBinary(db: Database, namespace: BinaryNamespace, id: string): Uint8Array<ArrayBuffer> | undefined;
+/**
+ * Bring blobs an earlier build stored more than once down to one copy. The
+ * caller runs each `step` in its own write transaction. A
+ * copy goes only when canonical storage holds the blob and it verifies:
+ *
+ * - main's whole row of a large blob that also has the chunk form;
+ * - a capture pin's own copy, which becomes a pin row that references the
+ *   canonical chunks (`captureBlob`).
+ *
+ * Verification reads a bounded number of chunks per call, so a large blob
+ * is verified across calls. The hash in progress lives in memory: after a
+ * restart that blob is verified again from its start. A canonical whole row
+ * up to 1 MiB verifies in one call; a larger one has no bounded read and is
+ * skipped, and so is canonical storage with a chunk past the blob's end. A
+ * candidate that fails verification is skipped for the compactor's life,
+ * and a step examines at most 8 candidates, so corrupt blobs cannot hold
+ * back the rest or stall a step. Once a
+ * search finds nothing, the compactor stops for good: current writes keep
+ * one copy, so only a database an earlier build wrote has work, and an idle
+ * step then reads none of these tables.
+ */
+export declare function createBinaryCompactor(db: Database): {
+    /**
+     * Spend up to about `bytes` of canonical reads, and at most
+     * `COMPACT_CANDIDATES_PER_STEP` new candidates, toward compaction. Call it
+     * in a write transaction.
+     */
+    step(bytes: number): void;
+};
+//# sourceMappingURL=pdpp-binary-chunks.d.ts.map

package/dist/storage/pdpp-binary-chunks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdpp-binary-chunks.d.ts","sourceRoot":"","sources":["../../src/storage/pdpp-binary-chunks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,eAAO,MAAM,kBAAkB,QAAY,CAAC;AAC5C,mFAAmF;AACnF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AACvD,eAAO,MAAM,cAAc,KAAK,CAAC;AACjC,eAAO,MAAM,sBAAsB,QAAc,CAAC;AAGlD,eAAO,MAAM,uBAAuB,4WAOnC,CAAC;AACF,MAAM,MAAM,eAAe,GACzB,WAAW,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;AAC1E;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB;;;;;;;CAO2C,CAAC;AAU9E,gFAAgF;AAChF,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,GACT,MAAM,GAAG,SAAS,CAmBpB;AAED,wBAAiB,gBAAgB,CAC/B,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,GACT,SAAS,CAAC,UAAU,CAAC,CAmEvB;AAED,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,MAAM,GACb;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,YAAY,CAAA;CAAE,CAuC3C;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,GACX,OAAO,CAST;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,GACT,IAAI,CAON;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,GACT,IAAI,CAeN;AAED,2FAA2F;AAC3F,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM;iBAOK,UAAU;;EAoB1B;AAED,wBAAgB,UAAU,CACxB,EAAE,EAAE,QAAQ,EACZ,IAAI,EAAE,eAAe,EACrB,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,eAAe,EACnB,QAAQ,SAAW,GAClB,IAAI,CAIN;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,QAAQ,EACZ,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,UAAU,GAChB,IAAI,CAmBN;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,QAAQ,EACZ,IAAI,EAAE,eAAe,EACrB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAAW,GAClB,IAAI,CAWN;AAED;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,CAAC,EAAE,EAAE,QAAQ,GAAG,MAAM,CAuCjE;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,eAAe,EAC1B,EAAE,EAAE,MAAM,GACT,UAAU,CAAC,WAAW,CAAC,GAAG,SAAS,CAcrC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ;IAuJ9C;;;;OAIG;gBACS,MAAM;EAerB"}

package/dist/storage/pdpp-binary-chunks.js

@@ -0,0 +1,480 @@
+import { createHash } from "node:crypto";
+export const BINARY_CHUNK_BYTES = 64 * 1024;
+/** Legacy whole-row reads: windows are at least this big and at most this many. */
+export const LEGACY_MIN_WINDOW_BYTES = 8 * 1024 * 1024;
+export const LEGACY_WINDOWS = 16;
+export const CANONICAL_INLINE_BYTES = 1024 * 1024;
+/** Candidates one compaction step may examine, so skipped ones cannot stall it. */
+const COMPACT_CANDIDATES_PER_STEP = 8;
+export const BINARY_CHUNKS_MIGRATION = `
+CREATE TABLE IF NOT EXISTS pdpp_binary_manifests (
+ namespace TEXT NOT NULL, blob_id TEXT NOT NULL, size_bytes INTEGER NOT NULL,
+ PRIMARY KEY(namespace,blob_id));
+CREATE TABLE IF NOT EXISTS pdpp_binary_chunks (
+ namespace TEXT NOT NULL, blob_id TEXT NOT NULL, position INTEGER NOT NULL, bytes BLOB NOT NULL,
+ PRIMARY KEY(namespace,blob_id,position));
+`;
+/**
+ * What each namespace's bytes are for. A "reader" namespace holds bytes a
+ * reader can be served. Each one MUST have manifest insert and delete
+ * triggers in the visibility view (pdpp-replica-visibility-view.ts), or
+ * records stay stale when its chunked bytes arrive or leave. Add a new
+ * namespace here first; pdpp-replica-invariants.test.ts guards the triggers.
+ */
+export const BINARY_NAMESPACE_ROLE = {
+    canonical: "reader",
+    received: "reader",
+    pending: "reader",
+    pinned: "export",
+    staging: "scratch",
+    payload: "scratch",
+};
+const legacy = {
+    canonical: "pdpp_blob_bytes",
+    pinned: "pdpp_replica_blob_bytes",
+    received: "pdpp_replica_received_bytes",
+    staging: null,
+    payload: null,
+    pending: null,
+};
+/** Read chunk copies when present; fall back to main's whole-value blob row. */
+export function binaryLength(db, namespace, id) {
+    const manifest = db
+        .prepare("SELECT size_bytes AS size FROM pdpp_binary_manifests WHERE namespace=? AND blob_id=?")
+        .get(namespace, id);
+    if (manifest)
+        return Number.isSafeInteger(manifest.size) && manifest.size >= 0
+            ? manifest.size
+            : undefined;
+    const table = legacy[namespace];
+    if (table) {
+        const row = db
+            .prepare(`SELECT length(bytes) AS size FROM ${table} WHERE blob_id=?`)
+            .get(id);
+        if (row)
+            return row.size;
+    }
+    return undefined;
+}
+export function* readBinaryChunks(db, namespace, id) {
+    const table = legacy[namespace];
+    const hasManifest = db
+        .prepare("SELECT 1 FROM pdpp_binary_manifests WHERE namespace=? AND blob_id=?")
+        .get(namespace, id) !== undefined;
+    if (table && !hasManifest) {
+        const row = db
+            .prepare(`SELECT length(bytes) AS size FROM ${table} WHERE blob_id=?`)
+            .get(id);
+        if (row) {
+            // SQLite reaches a BLOB offset by walking its overflow pages, so a read
+            // at offset N costs O(N). One read per 64 KiB chunk is quadratic (a
+            // 64 MiB row took 37 s). Reading windows of size/WINDOWS_PER_BLOB bounds
+            // the walk to a constant multiple of the blob and memory to a fraction.
+            const readWindow = db.prepare(`SELECT substr(bytes, ?, ?) AS bytes FROM ${table} WHERE blob_id=?`);
+            const window = Math.ceil(Math.max(LEGACY_MIN_WINDOW_BYTES, row.size / LEGACY_WINDOWS) /
+                BINARY_CHUNK_BYTES) * BINARY_CHUNK_BYTES;
+            for (let start = 0; start < row.size; start += window) {
+                const part = readWindow.get(start + 1, window, id);
+                if (!part?.bytes.length)
+                    throw new Error("corrupt binary chunks");
+                for (let offset = 0; offset < part.bytes.length; offset += BINARY_CHUNK_BYTES)
+                    yield part.bytes.subarray(offset, offset + BINARY_CHUNK_BYTES);
+            }
+            return;
+        }
+    }
+    const size = binaryLength(db, namespace, id);
+    if (size === undefined)
+        throw new Error("missing binary bytes");
+    const read = db.prepare("SELECT bytes FROM pdpp_binary_chunks WHERE namespace=? AND blob_id=? AND position=?");
+    let offset = 0, position = 0;
+    while (offset < size) {
+        const row = read.get(namespace, id, position++);
+        if (!row ||
+            !row.bytes.length ||
+            row.bytes.length > BINARY_CHUNK_BYTES ||
+            offset + row.bytes.length > size)
+            throw new Error("corrupt binary chunks");
+        offset += row.bytes.length;
+        yield row.bytes;
+    }
+    if (db
+        .prepare("SELECT 1 FROM pdpp_binary_chunks WHERE namespace=? AND blob_id=? AND position>=? LIMIT 1")
+        .get(namespace, id, position))
+        throw new Error("excess binary chunks");
+}
+/**
+ * Read exactly `length` bytes of chunked (manifest) bytes from `from`, and
+ * return them with the cursor after them. Rows may have any length, so a
+ * caller that reads a value in parts keeps the returned cursor; it never
+ * seeks by arithmetic. Memory is `length`. Legacy whole-row values are not
+ * supported: only namespaces without a legacy table use this.
+ */
+export function readBinaryWindow(db, namespace, id, from, length) {
+    if (legacy[namespace])
+        throw new Error("legacy binary windows unsupported");
+    const size = binaryLength(db, namespace, id);
+    if (size === undefined ||
+        !Number.isSafeInteger(length) ||
+        length < 1 ||
+        !Number.isSafeInteger(from.position) ||
+        from.position < 0 ||
+        !Number.isSafeInteger(from.offset) ||
+        from.offset < 0)
+        throw new Error("missing binary bytes");
+    const bytes = new Uint8Array(length);
+    const read = db.prepare("SELECT bytes FROM pdpp_binary_chunks WHERE namespace=? AND blob_id=? AND position=?");
+    let filled = 0;
+    let { position, offset } = from;
+    while (filled < length) {
+        const row = read.get(namespace, id, position);
+        if (!row ||
+            !row.bytes.length ||
+            row.bytes.length > BINARY_CHUNK_BYTES ||
+            offset >= row.bytes.length)
+            throw new Error("corrupt binary chunks");
+        const take = Math.min(row.bytes.length - offset, length - filled);
+        bytes.set(row.bytes.subarray(offset, offset + take), filled);
+        filled += take;
+        offset += take;
+        if (offset === row.bytes.length) {
+            position++;
+            offset = 0;
+        }
+    }
+    return { bytes, next: { position, offset } };
+}
+export function verifyBinary(db, namespace, id, digest, size) {
+    if (binaryLength(db, namespace, id) !== size)
+        return false;
+    const hash = createHash("sha256");
+    try {
+        for (const bytes of readBinaryChunks(db, namespace, id))
+            hash.update(bytes);
+    }
+    catch {
+        return false;
+    }
+    return `sha256:${hash.digest("hex")}` === digest;
+}
+export function deleteBinary(db, namespace, id) {
+    db.prepare("DELETE FROM pdpp_binary_chunks WHERE namespace=? AND blob_id=?").run(namespace, id);
+    db.prepare("DELETE FROM pdpp_binary_manifests WHERE namespace=? AND blob_id=?").run(namespace, id);
+}
+/**
+ * A repair write is about to give `id` a verified copy in `namespace`.
+ * Remove a published manifest whose size is not a safe non-negative
+ * integer, with its chunks, in the caller's transaction. Readers veto a
+ * namespace on such a manifest even beside a legacy row, so without this a
+ * repair would never take effect.
+ */
+export function clearMalformedManifest(db, namespace, id) {
+    const row = db
+        .prepare("SELECT size_bytes AS size FROM pdpp_binary_manifests WHERE namespace=? AND blob_id=?")
+        .get(namespace, id);
+    if (row &&
+        !(typeof row.size === "number" &&
+            Number.isSafeInteger(row.size) &&
+            row.size >= 0))
+        deleteBinary(db, namespace, id);
+}
+/** Publish the manifest only after EOF. Callers own transaction/visibility and cleanup. */
+export function binaryWriter(db, namespace, id) {
+    deleteBinary(db, namespace, id);
+    const insert = db.prepare("INSERT INTO pdpp_binary_chunks VALUES (?,?,?,?)");
+    let size = 0, position = 0;
+    return {
+        write(bytes) {
+            for (let offset = 0; offset < bytes.length; offset += BINARY_CHUNK_BYTES) {
+                const part = bytes.subarray(offset, offset + BINARY_CHUNK_BYTES);
+                insert.run(namespace, id, position++, part);
+                size += part.length;
+            }
+        },
+        finish() {
+            db.prepare("INSERT INTO pdpp_binary_manifests VALUES (?,?,?)").run(namespace, id, size);
+            return size;
+        },
+    };
+}
+export function copyBinary(db, from, sourceId, to, targetId = sourceId) {
+    const writer = binaryWriter(db, to, targetId);
+    for (const chunk of readBinaryChunks(db, from, sourceId))
+        writer.write(chunk);
+    writer.finish();
+}
+/**
+ * Write canonical bytes the server holds as one value, in one form only. A
+ * value up to CANONICAL_INLINE_BYTES (one Lite fragment) is main's whole row.
+ * A larger one is chunks only: the transport and capture's pin read those
+ * chunks, and the flag-off boot that retires replication writes main's row
+ * (`materializeCanonicalBinaries`).
+ */
+export function writeCanonicalBinary(db, id, bytes) {
+    deleteBinary(db, "canonical", id);
+    if (bytes.length > CANONICAL_INLINE_BYTES) {
+        db.prepare(`DELETE FROM ${legacy.canonical} WHERE blob_id=?`).run(id);
+        const writer = binaryWriter(db, "canonical", id);
+        for (let offset = 0; offset < bytes.length; offset += BINARY_CHUNK_BYTES)
+            writer.write(bytes.subarray(offset, offset + BINARY_CHUNK_BYTES));
+        writer.finish();
+        return;
+    }
+    const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    db.prepare(`INSERT INTO ${legacy.canonical} (blob_id, bytes) VALUES (?, ?)
+     ON CONFLICT(blob_id) DO UPDATE SET bytes = excluded.bytes`).run(id, view);
+}
+/**
+ * Promote verified bytes to canonical in the caller's transaction. A value up
+ * to CANONICAL_INLINE_BYTES (one Lite fragment) becomes main's whole row. A
+ * larger one is copied chunk by chunk and is stored only in the chunk form:
+ * no whole value is assembled or stored twice. Canonical readers accept both
+ * forms; the flag-off boot that retires replication writes main's row for
+ * chunk-only blobs (`materializeCanonicalBinaries`).
+ */
+export function copyBinaryToCanonical(db, from, sourceId, targetId = sourceId) {
+    const size = binaryLength(db, from, sourceId);
+    if (size === undefined)
+        throw new Error("missing binary bytes");
+    if (size > CANONICAL_INLINE_BYTES) {
+        db.prepare(`DELETE FROM ${legacy.canonical} WHERE blob_id=?`).run(targetId);
+        copyBinary(db, from, sourceId, "canonical", targetId);
+        return;
+    }
+    const bytes = readBinary(db, from, sourceId);
+    if (!bytes)
+        throw new Error("missing binary bytes");
+    writeCanonicalBinary(db, targetId, bytes);
+}
+/**
+ * Write main's whole row for each verified canonical blob that has only the
+ * chunk form, so a flag-off or plain main build can read it. One blob per
+ * transaction; each costs one whole value in memory. Run it only when
+ * replication is retired: an enabled store keeps large blobs chunk-only.
+ * A first enable that stopped after the door committed, before the replica
+ * schema, left no chunk store, so there is nothing to write.
+ */
+export function materializeCanonicalBinaries(db) {
+    const chunkStore = db
+        .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'pdpp_binary_manifests'")
+        .get();
+    if (!chunkStore)
+        return 0;
+    const missing = db
+        .prepare(`SELECT m.blob_id, b.size_bytes, b.sha256
+       FROM pdpp_binary_manifests m
+       JOIN pdpp_blobs b ON b.blob_id = m.blob_id
+       LEFT JOIN ${legacy.canonical} c ON c.blob_id = m.blob_id
+       WHERE m.namespace = 'canonical' AND c.blob_id IS NULL`)
+        .all();
+    let written = 0;
+    for (const blob of missing)
+        db.transaction(() => {
+            if (!verifyBinary(db, "canonical", blob.blob_id, `sha256:${blob.sha256}`, blob.size_bytes))
+                return;
+            const bytes = readBinary(db, "canonical", blob.blob_id);
+            if (!bytes)
+                return;
+            // Main's row becomes the only copy.
+            db.prepare(`INSERT INTO ${legacy.canonical} (blob_id, bytes) VALUES (?, ?)`).run(blob.blob_id, Buffer.from(bytes.buffer, 0, bytes.byteLength));
+            deleteBinary(db, "canonical", blob.blob_id);
+            written++;
+        }).immediate();
+    return written;
+}
+/**
+ * Explicit whole-value read for HTTP consumers, rollback materialization and
+ * inline-size promotion. Replication never calls it on a value larger than
+ * CANONICAL_INLINE_BYTES.
+ */
+export function readBinary(db, namespace, id) {
+    const length = binaryLength(db, namespace, id);
+    if (length === undefined)
+        return undefined;
+    const result = new Uint8Array(length);
+    let offset = 0;
+    try {
+        for (const bytes of readBinaryChunks(db, namespace, id)) {
+            result.set(bytes, offset);
+            offset += bytes.length;
+        }
+    }
+    catch {
+        return undefined;
+    }
+    return result;
+}
+/**
+ * Bring blobs an earlier build stored more than once down to one copy. The
+ * caller runs each `step` in its own write transaction. A
+ * copy goes only when canonical storage holds the blob and it verifies:
+ *
+ * - main's whole row of a large blob that also has the chunk form;
+ * - a capture pin's own copy, which becomes a pin row that references the
+ *   canonical chunks (`captureBlob`).
+ *
+ * Verification reads a bounded number of chunks per call, so a large blob
+ * is verified across calls. The hash in progress lives in memory: after a
+ * restart that blob is verified again from its start. A canonical whole row
+ * up to 1 MiB verifies in one call; a larger one has no bounded read and is
+ * skipped, and so is canonical storage with a chunk past the blob's end. A
+ * candidate that fails verification is skipped for the compactor's life,
+ * and a step examines at most 8 candidates, so corrupt blobs cannot hold
+ * back the rest or stall a step. Once a
+ * search finds nothing, the compactor stops for good: current writes keep
+ * one copy, so only a database an earlier build wrote has work, and an idle
+ * step then reads none of these tables.
+ */
+export function createBinaryCompactor(db) {
+    const known = "IN (SELECT blob_id FROM pdpp_blobs)";
+    /** Per kind: queries for its candidates after `@after`, first by blob id. */
+    const kinds = {
+        whole: {
+            arms: [
+                `SELECT c.blob_id FROM ${legacy.canonical} c
+         JOIN pdpp_binary_manifests m ON m.namespace='canonical' AND m.blob_id=c.blob_id
+         WHERE m.size_bytes > ${CANONICAL_INLINE_BYTES} AND c.blob_id>@after AND c.blob_id ${known}`,
+            ],
+            compact(id) {
+                db.prepare(`DELETE FROM ${legacy.canonical} WHERE blob_id=?`).run(id);
+            },
+        },
+        pinned: {
+            arms: [
+                `SELECT p.blob_id FROM pdpp_binary_manifests p
+         JOIN pdpp_binary_manifests m ON m.namespace='canonical' AND m.blob_id=p.blob_id
+         WHERE p.namespace='pinned' AND p.blob_id>@after AND p.blob_id ${known}`,
+            ],
+            compact(id) {
+                // Export may have released the pin since verification began.
+                if (binaryLength(db, "pinned", id) === undefined)
+                    return;
+                db.prepare("INSERT OR IGNORE INTO pdpp_replica_blob_pins VALUES (?)").run(id);
+                deleteBinary(db, "pinned", id);
+            },
+        },
+    };
+    const order = Object.keys(kinds);
+    /**
+     * Keyset position: the kind being searched and the last blob examined in
+     * it. A candidate is examined once per process, so a skipped one costs no
+     * later search.
+     */
+    let cursor = { kind: 0, after: "" };
+    let current;
+    /** The next candidate, or undefined when none is left. */
+    function next() {
+        while (cursor.kind < order.length) {
+            const kind = order[cursor.kind];
+            const row = db
+                .prepare(`SELECT min(blob_id) AS blob_id FROM (${kinds[kind].arms
+                .map((arm) => `SELECT * FROM (${arm} ORDER BY 1 LIMIT 1)`)
+                .join(" UNION ALL ")})`)
+                .get({ after: cursor.after });
+            if (row.blob_id !== null) {
+                cursor.after = row.blob_id;
+                return { kind, id: row.blob_id };
+            }
+            cursor = { kind: cursor.kind + 1, after: "" };
+        }
+        return undefined;
+    }
+    /**
+     * Begin verifying a candidate. A canonical whole row up to 1 MiB verifies
+     * and compacts at once; a larger whole row has no bounded read and is
+     * skipped. Returns the bytes read.
+     */
+    function begin(candidate) {
+        const meta = db
+            .prepare("SELECT size_bytes AS size, sha256 FROM pdpp_blobs WHERE blob_id=?")
+            .get(candidate.id);
+        const size = binaryLength(db, "canonical", candidate.id);
+        if (!meta || size !== meta.size)
+            return 0;
+        const digest = `sha256:${meta.sha256}`;
+        const chunked = db
+            .prepare("SELECT 1 FROM pdpp_binary_manifests WHERE namespace='canonical' AND blob_id=?")
+            .get(candidate.id);
+        if (!chunked) {
+            if (size > CANONICAL_INLINE_BYTES)
+                return 0;
+            if (verifyBinary(db, "canonical", candidate.id, digest, size))
+                kinds[candidate.kind].compact(candidate.id);
+            return size;
+        }
+        current = {
+            ...candidate,
+            size,
+            digest,
+            hash: createHash("sha256"),
+            position: 0,
+            read: 0,
+        };
+        return 0;
+    }
+    /**
+     * Hash up to `bytes` more of the current blob, at least one chunk, and
+     * compact it once it verifies. Returns the bytes read.
+     */
+    function advance(bytes) {
+        const blob = current;
+        const rows = db
+            .prepare("SELECT position, bytes FROM pdpp_binary_chunks WHERE namespace='canonical' AND blob_id=? AND position>=? ORDER BY position")
+            .iterate(blob.id, blob.position);
+        let read = 0;
+        let corrupt = true;
+        for (const row of rows) {
+            corrupt =
+                row.position !== blob.position ||
+                    !row.bytes.length ||
+                    row.bytes.length > BINARY_CHUNK_BYTES ||
+                    blob.read + row.bytes.length > blob.size;
+            if (corrupt || (read && read + row.bytes.length > bytes))
+                break;
+            blob.hash.update(row.bytes);
+            blob.position++;
+            blob.read += row.bytes.length;
+            read += row.bytes.length;
+            if (blob.read === blob.size)
+                break;
+        }
+        if (corrupt)
+            current = undefined;
+        else if (blob.read === blob.size) {
+            current = undefined;
+            // Readers reject a chunk past the blob's end, so verification does too.
+            const trailing = db
+                .prepare("SELECT 1 FROM pdpp_binary_chunks WHERE namespace='canonical' AND blob_id=? AND position>=?")
+                .get(blob.id, blob.position);
+            if (!trailing &&
+                `sha256:${blob.hash.digest("hex")}` === blob.digest &&
+                binaryLength(db, "canonical", blob.id) === blob.size)
+                kinds[blob.kind].compact(blob.id);
+        }
+        return Math.max(read, 1);
+    }
+    return {
+        /**
+         * Spend up to about `bytes` of canonical reads, and at most
+         * `COMPACT_CANDIDATES_PER_STEP` new candidates, toward compaction. Call it
+         * in a write transaction.
+         */
+        step(bytes) {
+            let started = 0;
+            while (bytes >= BINARY_CHUNK_BYTES) {
+                if (current) {
+                    bytes -= advance(bytes);
+                    continue;
+                }
+                if (started === COMPACT_CANDIDATES_PER_STEP)
+                    return;
+                const candidate = next();
+                if (!candidate)
+                    return;
+                started++;
+                bytes -= begin(candidate);
+            }
+        },
+    };
+}
+//# sourceMappingURL=pdpp-binary-chunks.js.map

package/dist/storage/pdpp-binary-chunks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdpp-binary-chunks.js","sourceRoot":"","sources":["../../src/storage/pdpp-binary-chunks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC5C,mFAAmF;AACnF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,CAAC;AACjC,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,GAAG,IAAI,CAAC;AAClD,mFAAmF;AACnF,MAAM,2BAA2B,GAAG,CAAC,CAAC;AACtC,MAAM,CAAC,MAAM,uBAAuB,GAAG;;;;;;;CAOtC,CAAC;AAGF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,SAAS,EAAE,QAAQ;IACnB,QAAQ,EAAE,QAAQ;IAClB,OAAO,EAAE,QAAQ;IACjB,MAAM,EAAE,QAAQ;IAChB,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,SAAS;CACyD,CAAC;AAC9E,MAAM,MAAM,GAAG;IACb,SAAS,EAAE,iBAAiB;IAC5B,MAAM,EAAE,yBAAyB;IACjC,QAAQ,EAAE,6BAA6B;IACvC,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;CACL,CAAC;AAEX,gFAAgF;AAChF,MAAM,UAAU,YAAY,CAC1B,EAAY,EACZ,SAA0B,EAC1B,EAAU;IAEV,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CACN,sFAAsF,CACvF;SACA,GAAG,CAAC,SAAS,EAAE,EAAE,CAAiC,CAAC;IACtD,IAAI,QAAQ;QACV,OAAO,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,IAAI,CAAC;YAC9D,CAAC,CAAC,QAAQ,CAAC,IAAI;YACf,CAAC,CAAC,SAAS,CAAC;IAEhB,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,EAAE;aACX,OAAO,CAAC,qCAAqC,KAAK,kBAAkB,CAAC;aACrE,GAAG,CAAC,EAAE,CAAiC,CAAC;QAC3C,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,gBAAgB,CAC/B,EAAY,EACZ,SAA0B,EAC1B,EAAU;IAEV,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,WAAW,GACf,EAAE;SACC,OAAO,CACN,qEAAqE,CACtE;SACA,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,SAAS,CAAC;IACtC,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,EAAE;aACX,OAAO,CAAC,qCAAqC,KAAK,kBAAkB,CAAC;aACrE,GAAG,CAAC,EAAE,CAAiC,CAAC;QAC3C,IAAI,GAAG,EAAE,CAAC;YACR,wEAAwE;YACxE,oEAAoE;YACpE,yEAAyE;YACzE,wEAAwE;YACxE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,4CAA4C,KAAK,kBAAkB,CACpE,CAAC;YACF,MAAM,MAAM,GACV,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC;gBAC1D,kBAAkB,CACrB,GAAG,kBAAkB,CAAC;YACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;gBACtD,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,CAClB,CAAC;gBAChC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM;oBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBAClE,KACE,IAAI,MAAM,GAAG,CAAC,EACd,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAC1B,MAAM,IAAI,kBAAkB;oBAE5B,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,CAAC,CAAC;YACnE,CAAC;YACD,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,qFAAqF,CACtF,CAAC;IACF,IAAI,MAAM,GAAG,CAAC,EACZ,QAAQ,GAAG,CAAC,CAAC;IACf,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,CACf,CAAC;QAChC,IACE,CAAC,GAAG;YACJ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM;YACjB,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB;YACrC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI;YAEhC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,GAAG,CAAC,KAAK,CAAC;IAClB,CAAC;IACD,IACE,EAAE;SACC,OAAO,CACN,0FAA0F,CAC3F;SACA,GAAG,CAAC,SAAS,EAAE,EAAE,EAAE,QAAQ,CAAC;QAE/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;AAC5C,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,EAAY,EACZ,SAA0B,EAC1B,EAAU,EACV,IAAkB,EAClB,MAAc;IAEd,IAAI,MAAM,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7C,IACE,IAAI,KAAK,SAAS;QAClB,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC;QAC7B,MAAM,GAAG,CAAC;QACV,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,CAAC;QACjB,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,CAAC;QAEf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,qFAAqF,CACtF,CAAC;IACF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAChC,OAAO,MAAM,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,EAAE,QAAQ,CACb,CAAC;QAChC,IACE,CAAC,GAAG;YACJ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM;YACjB,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB;YACrC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM;YAE1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;QAClE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,IAAI,IAAI,CAAC;QACf,MAAM,IAAI,IAAI,CAAC;QACf,IAAI,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAC;YACX,MAAM,GAAG,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,EAAY,EACZ,SAA0B,EAC1B,EAAU,EACV,MAAc,EACd,IAAY;IAEZ,IAAI,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC3D,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,MAAM,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,EAAY,EACZ,SAA0B,EAC1B,EAAU;IAEV,EAAE,CAAC,OAAO,CACR,gEAAgE,CACjE,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACrB,EAAE,CAAC,OAAO,CACR,mEAAmE,CACpE,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACvB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,EAAY,EACZ,SAA0B,EAC1B,EAAU;IAEV,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CACN,sFAAsF,CACvF;SACA,GAAG,CAAC,SAAS,EAAE,EAAE,CAAkC,CAAC;IACvD,IACE,GAAG;QACH,CAAC,CACC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;YAC5B,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;YAC9B,GAAG,CAAC,IAAI,IAAI,CAAC,CACd;QAED,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AACpC,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,YAAY,CAC1B,EAAY,EACZ,SAA0B,EAC1B,EAAU;IAEV,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAC7E,IAAI,IAAI,GAAG,CAAC,EACV,QAAQ,GAAG,CAAC,CAAC;IACf,OAAO;QACL,KAAK,CAAC,KAAiB;YACrB,KACE,IAAI,MAAM,GAAG,CAAC,EACd,MAAM,GAAG,KAAK,CAAC,MAAM,EACrB,MAAM,IAAI,kBAAkB,EAC5B,CAAC;gBACD,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,CAAC,CAAC;gBACjE,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC5C,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM;YACJ,EAAE,CAAC,OAAO,CAAC,kDAAkD,CAAC,CAAC,GAAG,CAChE,SAAS,EACT,EAAE,EACF,IAAI,CACL,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,EAAY,EACZ,IAAqB,EACrB,QAAgB,EAChB,EAAmB,EACnB,QAAQ,GAAG,QAAQ;IAEnB,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,gBAAgB,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC;QAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9E,MAAM,CAAC,MAAM,EAAE,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAAY,EACZ,EAAU,EACV,KAAiB;IAEjB,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC1C,EAAE,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,SAAS,kBAAkB,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;QACjD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,kBAAkB;YACtE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC;QACpE,MAAM,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO;IACT,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CACtB,KAAK,CAAC,MAAqB,EAC3B,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,CACjB,CAAC;IACF,EAAE,CAAC,OAAO,CACR,eAAe,MAAM,CAAC,SAAS;+DAC4B,CAC5D,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CACnC,EAAY,EACZ,IAAqB,EACrB,QAAgB,EAChB,QAAQ,GAAG,QAAQ;IAEnB,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAChE,IAAI,IAAI,GAAG,sBAAsB,EAAE,CAAC;QAClC,EAAE,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,SAAS,kBAAkB,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5E,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;QACtD,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACpD,oBAAoB,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,4BAA4B,CAAC,EAAY;IACvD,MAAM,UAAU,GAAG,EAAE;SAClB,OAAO,CACN,qFAAqF,CACtF;SACA,GAAG,EAAE,CAAC;IACT,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAG,EAAE;SACf,OAAO,CACN;;;mBAGa,MAAM,CAAC,SAAS;6DAC0B,CACxD;SACA,GAAG,EAA+D,CAAC;IACtE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,OAAO;QACxB,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,IACE,CAAC,YAAY,CACX,EAAE,EACF,WAAW,EACX,IAAI,CAAC,OAAO,EACZ,UAAU,IAAI,CAAC,MAAM,EAAE,EACvB,IAAI,CAAC,UAAU,CAChB;gBAED,OAAO;YACT,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,oCAAoC;YACpC,EAAE,CAAC,OAAO,CACR,eAAe,MAAM,CAAC,SAAS,iCAAiC,CACjE,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YACpE,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACjB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,EAAY,EACZ,SAA0B,EAC1B,EAAU;IAEV,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAC/C,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACzB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAY;IAChD,MAAM,KAAK,GAAG,qCAAqC,CAAC;IACpD,6EAA6E;IAC7E,MAAM,KAAK,GAAG;QACZ,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,yBAAyB,MAAM,CAAC,SAAS;;gCAEjB,sBAAsB,uCAAuC,KAAK,EAAE;aAC7F;YACD,OAAO,CAAC,EAAU;gBAChB,EAAE,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,SAAS,kBAAkB,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,CAAC;SACF;QACD,MAAM,EAAE;YACN,IAAI,EAAE;gBACJ;;yEAEiE,KAAK,EAAE;aACzE;YACD,OAAO,CAAC,EAAU;gBAChB,6DAA6D;gBAC7D,IAAI,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,SAAS;oBAAE,OAAO;gBACzD,EAAE,CAAC,OAAO,CACR,yDAAyD,CAC1D,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACV,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjC,CAAC;SACF;KACO,CAAC;IAEX,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAW,CAAC;IAC3C;;;;OAIG;IACH,IAAI,MAAM,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACpC,IAAI,OAUS,CAAC;IACd,0DAA0D;IAC1D,SAAS,IAAI;QACX,OAAO,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC;YACjC,MAAM,GAAG,GAAG,EAAE;iBACX,OAAO,CACN,wCAAwC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI;iBACrD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,GAAG,sBAAsB,CAAC;iBACzD,IAAI,CAAC,aAAa,CAAC,GAAG,CAC1B;iBACA,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAA+B,CAAC;YAC9D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBACzB,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3B,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;YACnC,CAAC;YACD,MAAM,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QAChD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD;;;;OAIG;IACH,SAAS,KAAK,CAAC,SAAqC;QAClD,MAAM,IAAI,GAAG,EAAE;aACZ,OAAO,CACN,mEAAmE,CACpE;aACA,GAAG,CAAC,SAAS,CAAC,EAAE,CAAiD,CAAC;QACrE,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,EAAE;aACf,OAAO,CACN,+EAA+E,CAChF;aACA,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,IAAI,GAAG,sBAAsB;gBAAE,OAAO,CAAC,CAAC;YAC5C,IAAI,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;gBAC3D,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,GAAG;YACR,GAAG,SAAS;YACZ,IAAI;YACJ,MAAM;YACN,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;YAC1B,QAAQ,EAAE,CAAC;YACX,IAAI,EAAE,CAAC;SACR,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD;;;OAGG;IACH,SAAS,OAAO,CAAC,KAAa;QAC5B,MAAM,IAAI,GAAG,OAAQ,CAAC;QACtB,MAAM,IAAI,GAAG,EAAE;aACZ,OAAO,CACN,4HAA4H,CAC7H;aACA,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAG/B,CAAC;QACH,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;gBACL,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ;oBAC9B,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM;oBACjB,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB;oBACrC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;YAC3C,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC;gBAAE,MAAM;YAChE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;YAC9B,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;gBAAE,MAAM;QACrC,CAAC;QACD,IAAI,OAAO;YAAE,OAAO,GAAG,SAAS,CAAC;aAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YACjC,OAAO,GAAG,SAAS,CAAC;YACpB,wEAAwE;YACxE,MAAM,QAAQ,GAAG,EAAE;iBAChB,OAAO,CACN,4FAA4F,CAC7F;iBACA,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,IACE,CAAC,QAAQ;gBACT,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM;gBACnD,YAAY,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI;gBAEpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO;QACL;;;;WAIG;QACH,IAAI,CAAC,KAAa;YAChB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,OAAO,KAAK,IAAI,kBAAkB,EAAE,CAAC;gBACnC,IAAI,OAAO,EAAE,CAAC;oBACZ,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;oBACxB,SAAS;gBACX,CAAC;gBACD,IAAI,OAAO,KAAK,2BAA2B;oBAAE,OAAO;gBACpD,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,SAAS;oBAAE,OAAO;gBACvB,OAAO,EAAE,CAAC;gBACV,KAAK,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}