@opendatalabs/personal-server-ts-core 0.0.1-canary.016adf9 → 0.0.1-canary.052d47e

package/dist/sync/workers/delete.d.ts

@@ -0,0 +1,163 @@
+import type { Logger } from "../../logger/index.js";
+import type { DataPointFeedPort, DataStoragePort, DeleteDataPort, PendingBlobDeletion, PendingBlobDeletionStore } from "../../ports/index.js";
+import { type ScopeDeletionTracker } from "../scope-deletions.js";
+/**
+ * Exact blob deletes sent per pass. vana-storage allows 30 DELETE requests
+ * per owner per 60 seconds. A DELETE request's own pass and the next sync
+ * cycle's retry pass can land inside one window, so each pass sends at most
+ * half the budget; whatever does not fit is queued and drained on later
+ * cycles, and a 429 simply re-queues its key.
+ */
+export declare const BLOB_DELETE_BATCH_SIZE = 15;
+export interface DeleteScopeDeps {
+    storage: DataStoragePort;
+    /** Data owner; absent only on an unconfigured host (no key material). */
+    serverOwner?: string;
+    /**
+     * Remote deletion (gateway tombstone + storage blobs). Absent when the host
+     * runs without sync (no owner key material): deletion is then local-only
+     * and reported as such, never silently "durable".
+     */
+    deleteData?: DeleteDataPort | null;
+    /** Exact-key retry markers for blob deletes that did not finish. */
+    pendingBlobDeletions?: PendingBlobDeletionStore;
+    /**
+     * Read-side deletion memory. A landed tombstone is recorded here so this
+     * replica's reads answer 410 immediately, before any sync cycle.
+     */
+    scopeDeletions?: ScopeDeletionTracker;
+    /**
+     * Deletion-aware registry lookup, re-read right after the tombstone attempt
+     * (a registration racing a "not-registered" answer) and by the retry to
+     * expand a marker whose tombstone version was not known.
+     */
+    dataPointFeed?: DataPointFeedPort;
+    logger: Logger;
+    now?: () => Date;
+}
+export type DeleteStepStatus = "ok" | "skipped" | "failed" | "deferred";
+export interface DeleteScopeStep {
+    status: DeleteStepStatus;
+    /** Why the step was skipped (e.g. "not-registered", "sync-disabled"). */
+    reason?: string;
+    /** Error message when status is "failed". */
+    error?: string;
+}
+export interface DeleteScopeResult {
+    scope: string;
+    /** DPv2 identity: keccak256(abi.encode(owner, scope)); null without an owner. */
+    dataPointId: string | null;
+    /**
+     * True once the gateway holds the tombstone (or never held the point at
+     * all): sync can no longer resurrect the scope on any replica. False when
+     * only the local copy was removed.
+     */
+    durable: boolean;
+    steps: {
+        gateway: DeleteScopeStep & {
+            /** Tombstone version at the gateway; null when it did not report one. */
+            version?: string | null;
+            deletedAt?: string | null;
+        };
+        /**
+         * Storage step. "ok": every blob key the tombstone covers is gone.
+         * "deferred": some keys are queued for later passes (rate-limit batch,
+         * or a tombstone version the gateway did not report). "failed": at
+         * least one delete errored; the failed keys are queued too.
+         */
+        storage: DeleteScopeStep & {
+            blobsDeleted?: number;
+            blobsMissing?: number;
+            blobsPending?: number;
+        };
+        local: DeleteScopeStep & {
+            deletedCount?: number;
+        };
+    };
+    /**
+     * True when the tombstone landed but blob deletion did not finish: exact
+     * retry markers were recorded and later sync cycles finish them.
+     */
+    pendingBlobDeletion: boolean;
+}
+export interface BlobDeletionPlan {
+    /**
+     * Exact keys of covered local entries that lie outside the registry range:
+     * an upload that landed under a key the registry never saw (crashed before
+     * registering, or rebased later) is still deleted data. Bounded by the
+     * local index size.
+     */
+    keys: string[];
+    /**
+     * Every registry version up to and including the tombstone's, as inclusive
+     * bounds. Versions are a dense, strictly increasing integer sequence per
+     * scope and an absent key answers 404, so the range is exact and complete
+     * across replicas without listing storage. Kept as bounds, never
+     * materialised: a large tombstone version costs nothing until its keys are
+     * actually sent, a batch at a time. Null without a tombstone version.
+     */
+    range: {
+        from: string;
+        to: string;
+    } | null;
+}
+/**
+ * The storage keys a tombstone covers for `scope`. A re-add registered after
+ * the tombstone lives under a higher version key, so it is never in this set
+ * and survives by construction. With no tombstone version (never registered,
+ * or the gateway did not report one) only the local entries' keys can be
+ * enumerated.
+ */
+export declare function planBlobDeletions(storage: Pick<DataStoragePort, "listVersions">, scope: string, tombstoneVersionValue: string | null): BlobDeletionPlan;
+/**
+ * Durable scope deletion, in the only order that cannot resurrect data:
+ *
+ *   1. gateway tombstone  (owner-signed AddData at current + 1; the fact)
+ *   2. storage blobs      (exact keys of every version the tombstone covers)
+ *   3. local copy         (index rows + files + sidecars)
+ *
+ * The local delete never runs before the gateway has acknowledged: until the
+ * registry says "deleted", the next sync cycle would re-pull the point from
+ * the still-live row. Step 2 deletes exact per-version keys, never the scope
+ * prefix, so a re-add another replica lands at any time keeps its ciphertext
+ * (it is registered above the tombstone version and stored under that key).
+ * Keys that could not be deleted in this pass are queued as exact retry
+ * markers; step 3 does not wait for them because the tombstone already makes
+ * the deletion stick.
+ *
+ * Not serialised here: the sync manager runs this under the same lock as its
+ * upload/download cycles so a delete cannot interleave with an upload of the
+ * same scope on this replica (see engine/sync-manager.ts). Uploads racing on
+ * another replica are handled by the upload worker's deletion guard and the
+ * download worker's reconcile.
+ */
+export declare function deleteScope(deps: DeleteScopeDeps, scope: string): Promise<DeleteScopeResult>;
+export interface RetryPendingBlobDeletionsResult {
+    /** Exact keys finished this pass (deleted, or already absent). */
+    completed: PendingBlobDeletion[];
+    /** Unexpanded markers dropped because the scope is live again. */
+    superseded: string[];
+    failed: Array<{
+        scope: string;
+        version: string | null;
+        error: string;
+    }>;
+    /** Markers still queued after this pass (a range counts once). */
+    remaining: number;
+}
+/**
+ * Finish blob deletions whose tombstone landed but whose storage deletes did
+ * not: one batch of exact keys per sync cycle, taken from exact-key markers
+ * first and then from the head of range markers (which advance in place),
+ * each key retried until storage acknowledges (2xx or 404). Unexpanded
+ * markers (the gateway did not report a tombstone version at delete time)
+ * are expanded from the registry first; if the registry shows the scope live
+ * again the old tombstone's version is unknowable, so the marker is dropped
+ * and the pre-tombstone ciphertext stays (owner-decryptable only) until the
+ * next delete of that scope.
+ */
+export declare function retryPendingBlobDeletions(deps: Pick<DeleteScopeDeps, "deleteData" | "pendingBlobDeletions" | "dataPointFeed" | "serverOwner" | "logger"> & {
+    /** Local index, for expanding unexpanded markers into exact keys. */
+    storage?: Pick<DataStoragePort, "listVersions">;
+}): Promise<RetryPendingBlobDeletionsResult>;
+//# sourceMappingURL=delete.d.ts.map

package/dist/sync/workers/delete.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/delete.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EACV,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAEzC,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACnC,oEAAoE;IACpE,oBAAoB,CAAC,EAAE,wBAAwB,CAAC;IAChD;;;OAGG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC;;;;OAIG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,MAAM,MAAM,gBAAgB,GAAG,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,CAAC;AAExE,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,gBAAgB,CAAC;IACzB,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE;QACL,OAAO,EAAE,eAAe,GAAG;YACzB,yEAAyE;YACzE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;YACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;SAC3B,CAAC;QACF;;;;;WAKG;QACH,OAAO,EAAE,eAAe,GAAG;YACzB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,YAAY,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,KAAK,EAAE,eAAe,GAAG;YAAE,YAAY,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAAC;IACF;;;OAGG;IACH,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;;;;;;OAOG;IACH,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,cAAc,CAAC,EAC9C,KAAK,EAAE,MAAM,EACb,qBAAqB,EAAE,MAAM,GAAG,IAAI,GACnC,gBAAgB,CAqBlB;AAyCD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,eAAe,EACrB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,iBAAiB,CAAC,CAiJ5B;AA2FD,MAAM,WAAW,+BAA+B;IAC9C,kEAAkE;IAClE,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,kEAAkE;IAClE,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxE,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,IAAI,CACR,eAAe,EACb,YAAY,GACZ,sBAAsB,GACtB,eAAe,GACf,aAAa,GACb,QAAQ,CACX,GAAG;IACF,qEAAqE;IACrE,OAAO,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;CACjD,GACA,OAAO,CAAC,+BAA+B,CAAC,CAyJ1C"}

package/dist/sync/workers/delete.js

@@ -0,0 +1,474 @@
+import { computeDataPointId } from "../data-point-id.js";
+import { deletionTimestamp, isEntryCoveredByTombstone, tombstoneVersion, } from "../scope-deletions.js";
+/**
+ * Exact blob deletes sent per pass. vana-storage allows 30 DELETE requests
+ * per owner per 60 seconds. A DELETE request's own pass and the next sync
+ * cycle's retry pass can land inside one window, so each pass sends at most
+ * half the budget; whatever does not fit is queued and drained on later
+ * cycles, and a 429 simply re-queues its key.
+ */
+export const BLOB_DELETE_BATCH_SIZE = 15;
+/**
+ * The storage keys a tombstone covers for `scope`. A re-add registered after
+ * the tombstone lives under a higher version key, so it is never in this set
+ * and survives by construction. With no tombstone version (never registered,
+ * or the gateway did not report one) only the local entries' keys can be
+ * enumerated.
+ */
+export function planBlobDeletions(storage, scope, tombstoneVersionValue) {
+    const tombstone = { version: tombstoneVersionValue };
+    const last = tombstoneVersionValue === null ? null : BigInt(tombstoneVersionValue);
+    const keys = new Set();
+    const PAGE_SIZE = 500;
+    for (let offset = 0;; offset += PAGE_SIZE) {
+        const entries = storage.listVersions(scope, { limit: PAGE_SIZE, offset });
+        for (const entry of entries) {
+            if (!isEntryCoveredByTombstone(entry, tombstone))
+                continue;
+            const version = BigInt(entry.version);
+            if (last !== null && version >= 1n && version <= last)
+                continue;
+            keys.add(version.toString());
+        }
+        if (entries.length < PAGE_SIZE)
+            break;
+    }
+    return {
+        keys: [...keys].sort((a, b) => (BigInt(a) < BigInt(b) ? -1 : 1)),
+        range: last === null || last < 1n ? null : { from: "1", to: last.toString() },
+    };
+}
+/** Take up to `count` versions from the head of a range; `rest` is what remains. */
+function takeFromRange(range, count) {
+    const from = BigInt(range.from);
+    const to = BigInt(range.to);
+    const versions = [];
+    let cursor = from;
+    while (cursor <= to && versions.length < count) {
+        versions.push(cursor.toString());
+        cursor += 1n;
+    }
+    return {
+        versions,
+        rest: cursor <= to ? { from: cursor.toString(), to: range.to } : null,
+    };
+}
+function rangeSize(range) {
+    return BigInt(range.to) - BigInt(range.from) + 1n;
+}
+/**
+ * Count of blob keys a marker list still stands for, clamped for reporting.
+ * An unexpanded marker (range not known yet) counts as one.
+ */
+function countPendingKeys(markers) {
+    let total = 0n;
+    for (const marker of markers) {
+        if (marker.version !== null)
+            total += 1n;
+        else if (marker.range)
+            total += rangeSize(marker.range);
+        else
+            total += 1n;
+    }
+    return total > BigInt(Number.MAX_SAFE_INTEGER)
+        ? Number.MAX_SAFE_INTEGER
+        : Number(total);
+}
+/**
+ * Durable scope deletion, in the only order that cannot resurrect data:
+ *
+ *   1. gateway tombstone  (owner-signed AddData at current + 1; the fact)
+ *   2. storage blobs      (exact keys of every version the tombstone covers)
+ *   3. local copy         (index rows + files + sidecars)
+ *
+ * The local delete never runs before the gateway has acknowledged: until the
+ * registry says "deleted", the next sync cycle would re-pull the point from
+ * the still-live row. Step 2 deletes exact per-version keys, never the scope
+ * prefix, so a re-add another replica lands at any time keeps its ciphertext
+ * (it is registered above the tombstone version and stored under that key).
+ * Keys that could not be deleted in this pass are queued as exact retry
+ * markers; step 3 does not wait for them because the tombstone already makes
+ * the deletion stick.
+ *
+ * Not serialised here: the sync manager runs this under the same lock as its
+ * upload/download cycles so a delete cannot interleave with an upload of the
+ * same scope on this replica (see engine/sync-manager.ts). Uploads racing on
+ * another replica are handled by the upload worker's deletion guard and the
+ * download worker's reconcile.
+ */
+export async function deleteScope(deps, scope) {
+    const { storage, deleteData, pendingBlobDeletions, scopeDeletions, logger } = deps;
+    const now = deps.now ?? (() => new Date());
+    const dataPointId = deps.serverOwner
+        ? computeDataPointId(deps.serverOwner, scope)
+        : null;
+    const result = {
+        scope,
+        dataPointId,
+        durable: false,
+        steps: {
+            gateway: { status: "skipped", reason: "sync-disabled" },
+            storage: { status: "skipped", reason: "sync-disabled" },
+            local: { status: "skipped" },
+        },
+        pendingBlobDeletion: false,
+    };
+    if (deleteData) {
+        // 1. Gateway tombstone: the durable fact. Nothing else happens if it
+        // fails, so the owner gets a clear "nothing was deleted" instead of a
+        // local delete that sync silently undoes.
+        let tombstoneVersionValue = null;
+        let tombstoneKnown = false;
+        try {
+            let outcome = await deleteData.tombstone(scope);
+            if (outcome.status === "not-registered") {
+                // "Nothing to tombstone" is only durable if the registry still has
+                // no row. A registration racing our lookup would otherwise survive
+                // the local delete and sync would restore the scope: re-read, and
+                // tombstone the row that appeared. If it keeps appearing, or the
+                // registry cannot be asked, refuse rather than delete on a guess.
+                let registry = await registryState(deps, scope);
+                if (registry.status === "live") {
+                    outcome = await deleteData.tombstone(scope);
+                    registry = await registryState(deps, scope);
+                }
+                if (outcome.status === "not-registered" &&
+                    registry.status !== "deleted-or-absent") {
+                    throw registry.status === "unknown"
+                        ? registry.error
+                        : new Error("Scope was registered concurrently while it was being deleted; retry the delete");
+                }
+            }
+            if (outcome.status === "not-registered") {
+                result.steps.gateway = { status: "skipped", reason: "not-registered" };
+            }
+            else {
+                tombstoneKnown = true;
+                tombstoneVersionValue =
+                    outcome.version === null
+                        ? null
+                        : tombstoneVersion({ expectedVersion: outcome.version });
+                result.steps.gateway = {
+                    status: "ok",
+                    ...(outcome.status === "already-deleted" && {
+                        reason: "already-deleted",
+                    }),
+                    version: outcome.version,
+                    deletedAt: outcome.deletedAt,
+                };
+                // Reads on this replica must refuse the scope from this point on,
+                // without waiting for the feed to echo the tombstone back. A missing
+                // `deletedAt` (gateway did not echo one) is taken as "now": later
+                // than any local entry the deletion covers, so nothing slips through.
+                scopeDeletions?.markDeleted(scope, {
+                    deletedAt: outcome.deletedAt ?? now().toISOString(),
+                    version: outcome.version,
+                }, "local-delete");
+            }
+            result.durable = true;
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            result.steps.gateway = { status: "failed", error: message };
+            result.steps.storage = { status: "skipped", reason: "gateway-failed" };
+            result.steps.local = { status: "skipped", reason: "gateway-failed" };
+            logger.error({ scope, dataPointId, error: message }, "Gateway tombstone failed; scope NOT deleted (local copy kept so sync cannot resurrect a half-deleted scope)");
+            return result;
+        }
+        // 2. Storage blobs: exact keys, one pass of at most a batch, the rest
+        // queued. A tombstone whose version the gateway did not report cannot be
+        // expanded into registry keys yet; a version-less marker asks the retry
+        // to expand it once the registry answers.
+        const plan = planBlobDeletions(storage, scope, tombstoneVersionValue);
+        const batch = plan.keys.slice(0, BLOB_DELETE_BATCH_SIZE);
+        const leftovers = plan.keys
+            .slice(BLOB_DELETE_BATCH_SIZE)
+            .map((version) => ({ scope, version }));
+        if (plan.range) {
+            const taken = takeFromRange(plan.range, BLOB_DELETE_BATCH_SIZE - batch.length);
+            batch.push(...taken.versions);
+            if (taken.rest)
+                leftovers.push({ scope, version: null, range: taken.rest });
+        }
+        if (tombstoneKnown && tombstoneVersionValue === null) {
+            leftovers.push({ scope, version: null });
+        }
+        const storageStep = await deleteBlobKeys({ deleteData, pendingBlobDeletions, logger }, scope, batch, leftovers);
+        result.steps.storage = storageStep.step;
+        result.pendingBlobDeletion = storageStep.pending > 0;
+    }
+    // 3. Local copy. Safe now: the registry row is a tombstone (or the point
+    // was never registered / sync is off), so nothing can re-pull it.
+    try {
+        const deletedCount = await storage.deleteScope(scope);
+        result.steps.local = { status: "ok", deletedCount };
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        result.steps.local = { status: "failed", error: message };
+        logger.error({ scope, error: message }, "Local scope deletion failed");
+    }
+    logger.info({
+        scope,
+        dataPointId,
+        durable: result.durable,
+        gateway: result.steps.gateway.status,
+        storage: result.steps.storage.status,
+        local: result.steps.local.status,
+    }, "Scope deletion finished");
+    return result;
+}
+/**
+ * Delete one pass worth of exact keys and reconcile the retry markers:
+ * completed keys (deleted or already absent) leave the store, failed and
+ * not-yet-attempted keys enter it.
+ */
+async function deleteBlobKeys(deps, scope, batch, leftovers) {
+    const { deleteData, pendingBlobDeletions, logger } = deps;
+    let outcome;
+    try {
+        outcome =
+            batch.length > 0 && deleteData
+                ? await deleteData.deleteBlobVersions(scope, batch)
+                : { deleted: [], missing: [], failed: [] };
+    }
+    catch (err) {
+        // The port reports per-key failures; a thrown error means none of the
+        // batch can be trusted as done.
+        outcome = {
+            deleted: [],
+            missing: [],
+            failed: batch.map((version) => ({ version, error: errorMessage(err) })),
+        };
+    }
+    const completed = [...outcome.deleted, ...outcome.missing].map((version) => ({ scope, version }));
+    const remaining = [
+        ...outcome.failed.map(({ version }) => ({ scope, version })),
+        ...leftovers,
+    ];
+    const remainingKeys = countPendingKeys(remaining);
+    let recorded = remainingKeys;
+    if (pendingBlobDeletions) {
+        try {
+            if (completed.length > 0)
+                await pendingBlobDeletions.remove(completed);
+            if (remaining.length > 0)
+                await pendingBlobDeletions.add(remaining);
+        }
+        catch (markerErr) {
+            recorded = 0;
+            logger.error({ scope, error: errorMessage(markerErr), keys: remainingKeys }, "Could not record pending blob deletion markers");
+        }
+    }
+    else if (remaining.length > 0) {
+        recorded = 0;
+        logger.error({ scope, keys: remainingKeys }, "Blob deletions left unfinished with no marker store to retry them");
+    }
+    const counts = {
+        blobsDeleted: outcome.deleted.length,
+        blobsMissing: outcome.missing.length,
+        blobsPending: remainingKeys,
+    };
+    if (outcome.failed.length > 0) {
+        const first = outcome.failed[0];
+        logger.warn({
+            scope,
+            failed: outcome.failed.length,
+            pending: recorded,
+            error: first.error,
+        }, "Storage blob deletion failed for some keys after gateway tombstone; will retry");
+        return {
+            step: {
+                status: "failed",
+                error: `${outcome.failed.length} blob delete(s) failed: ${first.error}`,
+                ...counts,
+            },
+            pending: recorded,
+        };
+    }
+    if (remaining.length > 0) {
+        logger.info({ scope, ...counts }, "Storage blob deletion continues on later sync cycles (rate-limited batch)");
+        return { step: { status: "deferred", ...counts }, pending: recorded };
+    }
+    return { step: { status: "ok", ...counts }, pending: 0 };
+}
+/**
+ * Finish blob deletions whose tombstone landed but whose storage deletes did
+ * not: one batch of exact keys per sync cycle, taken from exact-key markers
+ * first and then from the head of range markers (which advance in place),
+ * each key retried until storage acknowledges (2xx or 404). Unexpanded
+ * markers (the gateway did not report a tombstone version at delete time)
+ * are expanded from the registry first; if the registry shows the scope live
+ * again the old tombstone's version is unknowable, so the marker is dropped
+ * and the pre-tombstone ciphertext stays (owner-decryptable only) until the
+ * next delete of that scope.
+ */
+export async function retryPendingBlobDeletions(deps) {
+    const { deleteData, pendingBlobDeletions, logger } = deps;
+    const result = {
+        completed: [],
+        superseded: [],
+        failed: [],
+        remaining: 0,
+    };
+    if (!deleteData || !pendingBlobDeletions)
+        return result;
+    let markers = await pendingBlobDeletions.list();
+    if (markers.length === 0)
+        return result;
+    // Expand markers whose tombstone version was unknown at delete time.
+    for (const marker of markers.filter((key) => key.version === null && !key.range)) {
+        const registry = await registryState(deps, marker.scope);
+        if (registry.status === "unknown") {
+            result.failed.push({
+                scope: marker.scope,
+                version: null,
+                error: registry.error.message,
+            });
+            continue;
+        }
+        if (registry.status === "live") {
+            await pendingBlobDeletions.remove([marker]);
+            result.superseded.push(marker.scope);
+            logger.warn({
+                scope: marker.scope,
+                dataPointId: registry.record.id,
+                version: registry.record.expectedVersion,
+            }, "Scope was re-added after its tombstone; dropping the unexpanded blob deletion marker so the live version's ciphertext survives");
+            continue;
+        }
+        const plan = planBlobDeletions(deps.storage ?? { listVersions: () => [] }, marker.scope, registry.status === "deleted" ? registry.version : null);
+        const expanded = plan.keys.map((version) => ({
+            scope: marker.scope,
+            version,
+        }));
+        if (plan.range) {
+            expanded.push({ scope: marker.scope, version: null, range: plan.range });
+        }
+        await pendingBlobDeletions.remove([marker]);
+        await pendingBlobDeletions.add(expanded);
+    }
+    markers = await pendingBlobDeletions.list();
+    // One rate-limited batch in stored order: exact keys, then range heads.
+    let budget = BLOB_DELETE_BATCH_SIZE;
+    const byScope = new Map();
+    const exactOrigin = new Set();
+    const advancedRanges = [];
+    const enqueue = (scope, version) => {
+        const versions = byScope.get(scope) ?? [];
+        versions.push(version);
+        byScope.set(scope, versions);
+    };
+    for (const marker of markers) {
+        if (budget === 0)
+            break;
+        if (marker.version !== null) {
+            enqueue(marker.scope, marker.version);
+            exactOrigin.add(`${marker.scope}\u0000${marker.version}`);
+            budget -= 1;
+        }
+    }
+    for (const marker of markers) {
+        if (budget === 0)
+            break;
+        if (marker.version === null && marker.range) {
+            const taken = takeFromRange(marker.range, budget);
+            for (const version of taken.versions)
+                enqueue(marker.scope, version);
+            budget -= taken.versions.length;
+            advancedRanges.push({
+                old: marker,
+                next: taken.rest
+                    ? { scope: marker.scope, version: null, range: taken.rest }
+                    : null,
+            });
+        }
+    }
+    for (const [scope, versions] of byScope) {
+        let outcome;
+        try {
+            outcome = await deleteData.deleteBlobVersions(scope, versions);
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            outcome = {
+                deleted: [],
+                missing: [],
+                failed: versions.map((version) => ({ version, error: message })),
+            };
+        }
+        const completed = [...outcome.deleted, ...outcome.missing]
+            .filter((version) => exactOrigin.has(`${scope}\u0000${version}`))
+            .map((version) => ({ scope, version }));
+        if (completed.length > 0)
+            await pendingBlobDeletions.remove(completed);
+        result.completed.push(...[...outcome.deleted, ...outcome.missing].map((version) => ({ scope, version })));
+        // A failed key that came from a range becomes an exact marker; a failed
+        // exact marker simply stays.
+        const failedFromRange = outcome.failed
+            .filter(({ version }) => !exactOrigin.has(`${scope}\u0000${version}`))
+            .map(({ version }) => ({ scope, version }));
+        if (failedFromRange.length > 0) {
+            await pendingBlobDeletions.add(failedFromRange);
+        }
+        for (const failure of outcome.failed) {
+            result.failed.push({ scope, ...failure });
+        }
+        if (outcome.deleted.length + outcome.missing.length > 0) {
+            logger.info({
+                scope,
+                deleted: outcome.deleted.length,
+                missing: outcome.missing.length,
+            }, "Completed pending blob deletions");
+        }
+        if (outcome.failed.length > 0) {
+            logger.warn({
+                scope,
+                failed: outcome.failed.length,
+                error: outcome.failed[0].error,
+            }, "Pending blob deletion failed again");
+        }
+    }
+    // Advance the ranges only now: a crash mid-pass re-sends already-deleted
+    // keys (404, harmless) rather than skipping keys that were never sent.
+    for (const { old, next } of advancedRanges) {
+        await pendingBlobDeletions.remove([old]);
+        if (next)
+            await pendingBlobDeletions.add([next]);
+    }
+    result.remaining = (await pendingBlobDeletions.list()).length;
+    return result;
+}
+/**
+ * What the registry currently says about `scope`. "live" means a row exists
+ * that is not a tombstone; "deleted" carries the tombstone's version;
+ * "deleted-or-absent" is used when nothing can be asked (no feed or owner
+ * wired) or the row is gone.
+ */
+async function registryState(deps, scope) {
+    if (!deps.dataPointFeed || !deps.serverOwner) {
+        return { status: "deleted-or-absent" };
+    }
+    let record;
+    try {
+        record = await deps.dataPointFeed.getDataPoint({
+            ownerAddress: deps.serverOwner,
+            scope,
+        });
+    }
+    catch (err) {
+        return {
+            status: "unknown",
+            error: err instanceof Error ? err : new Error(String(err)),
+        };
+    }
+    if (record === null)
+        return { status: "deleted-or-absent" };
+    if (deletionTimestamp(record) === null)
+        return { status: "live", record };
+    return { status: "deleted", version: tombstoneVersion(record) };
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+//# sourceMappingURL=delete.js.map

package/dist/sync/workers/delete.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/sync/workers/delete.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EACL,iBAAiB,EACjB,yBAAyB,EACzB,gBAAgB,GAEjB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AA8FzC;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAA8C,EAC9C,KAAa,EACb,qBAAoC;IAEpC,MAAM,SAAS,GAAG,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACrD,MAAM,IAAI,GACR,qBAAqB,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,SAAS,GAAG,GAAG,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,GAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC;gBAAE,SAAS;YAC3D,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,IAAI,IAAI;gBAAE,SAAS;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS;YAAE,MAAM;IACxC,CAAC;IACD,OAAO;QACL,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,KAAK,EACH,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;KACzE,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,aAAa,CACpB,KAAmC,EACnC,KAAa;IAEb,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,OAAO,MAAM,IAAI,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,EAAE,CAAC;IACf,CAAC;IACD,OAAO;QACL,QAAQ;QACR,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAmC;IACpD,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;AACpD,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAA8B;IACtD,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI;YAAE,KAAK,IAAI,EAAE,CAAC;aACpC,IAAI,MAAM,CAAC,KAAK;YAAE,KAAK,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;;YACnD,KAAK,IAAI,EAAE,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAC5C,CAAC,CAAC,MAAM,CAAC,gBAAgB;QACzB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAqB,EACrB,KAAa;IAEb,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,EAAE,GACzE,IAAI,CAAC;IACP,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW;QAClC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC;QAC7C,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,MAAM,GAAsB;QAChC,KAAK;QACL,WAAW;QACX,OAAO,EAAE,KAAK;QACd,KAAK,EAAE;YACL,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE;YACvD,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE;YACvD,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;SAC7B;QACD,mBAAmB,EAAE,KAAK;KAC3B,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,qEAAqE;QACrE,sEAAsE;QACtE,0CAA0C;QAC1C,IAAI,qBAAqB,GAAkB,IAAI,CAAC;QAChD,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,OAAO,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;gBACxC,mEAAmE;gBACnE,mEAAmE;gBACnE,kEAAkE;gBAClE,iEAAiE;gBACjE,kEAAkE;gBAClE,IAAI,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;oBAC/B,OAAO,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oBAC5C,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC9C,CAAC;gBACD,IACE,OAAO,CAAC,MAAM,KAAK,gBAAgB;oBACnC,QAAQ,CAAC,MAAM,KAAK,mBAAmB,EACvC,CAAC;oBACD,MAAM,QAAQ,CAAC,MAAM,KAAK,SAAS;wBACjC,CAAC,CAAC,QAAQ,CAAC,KAAK;wBAChB,CAAC,CAAC,IAAI,KAAK,CACP,gFAAgF,CACjF,CAAC;gBACR,CAAC;YACH,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;gBACxC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;YACzE,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,IAAI,CAAC;gBACtB,qBAAqB;oBACnB,OAAO,CAAC,OAAO,KAAK,IAAI;wBACtB,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,gBAAgB,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7D,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG;oBACrB,MAAM,EAAE,IAAI;oBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,iBAAiB,IAAI;wBAC1C,MAAM,EAAE,iBAAiB;qBAC1B,CAAC;oBACF,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,SAAS,EAAE,OAAO,CAAC,SAAS;iBAC7B,CAAC;gBACF,kEAAkE;gBAClE,qEAAqE;gBACrE,kEAAkE;gBAClE,sEAAsE;gBACtE,cAAc,EAAE,WAAW,CACzB,KAAK,EACL;oBACE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE;oBACnD,OAAO,EAAE,OAAO,CAAC,OAAO;iBACzB,EACD,cAAc,CACf,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;YACvE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;YACrE,MAAM,CAAC,KAAK,CACV,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,EACtC,6GAA6G,CAC9G,CAAC;YACF,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,sEAAsE;QACtE,yEAAyE;QACzE,wEAAwE;QACxE,0CAA0C;QAC1C,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACzD,MAAM,SAAS,GAA0B,IAAI,CAAC,IAAI;aAC/C,KAAK,CAAC,sBAAsB,CAAC;aAC7B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,aAAa,CACzB,IAAI,CAAC,KAAK,EACV,sBAAsB,GAAG,KAAK,CAAC,MAAM,CACtC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI;gBACZ,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,cAAc,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;YACrD,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,cAAc,CACtC,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAC5C,KAAK,EACL,KAAK,EACL,SAAS,CACV,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;QACxC,MAAM,CAAC,mBAAmB,GAAG,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,yEAAyE;IACzE,kEAAkE;IAClE,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC1D,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CACT;QACE,KAAK;QACL,WAAW;QACX,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;QACpC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;QACpC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;KACjC,EACD,yBAAyB,CAC1B,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC3B,IAA6E,EAC7E,KAAa,EACb,KAAe,EACf,SAAgC;IAEhC,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1D,IAAI,OAAkE,CAAC;IACvE,IAAI,CAAC;QACH,OAAO;YACL,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU;gBAC5B,CAAC,CAAC,MAAM,UAAU,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC;gBACnD,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACjD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,sEAAsE;QACtE,gCAAgC;QAChC,OAAO,GAAG;YACR,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SACxE,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAC5D,CAAC,OAAO,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CACvD,CAAC;IACF,MAAM,SAAS,GAA0B;QACvC,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5D,GAAG,SAAS;KACb,CAAC;IACF,MAAM,aAAa,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAClD,IAAI,QAAQ,GAAG,aAAa,CAAC;IAC7B,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,oBAAoB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,QAAQ,GAAG,CAAC,CAAC;YACb,MAAM,CAAC,KAAK,CACV,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAC9D,gDAAgD,CACjD,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,QAAQ,GAAG,CAAC,CAAC;QACb,MAAM,CAAC,KAAK,CACV,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,EAC9B,mEAAmE,CACpE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG;QACb,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;QACpC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;QACpC,YAAY,EAAE,aAAa;KAC5B,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CACT;YACE,KAAK;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;YAC7B,OAAO,EAAE,QAAQ;YACjB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,EACD,gFAAgF,CACjF,CAAC;QACF,OAAO;YACL,IAAI,EAAE;gBACJ,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,2BAA2B,KAAK,CAAC,KAAK,EAAE;gBACvE,GAAG,MAAM;aACV;YACD,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CACT,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,EACpB,2EAA2E,CAC5E,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACxE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;AAC3D,CAAC;AAYD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAUC;IAED,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1D,MAAM,MAAM,GAAoC;QAC9C,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,CAAC;KACb,CAAC;IACF,IAAI,CAAC,UAAU,IAAI,CAAC,oBAAoB;QAAE,OAAO,MAAM,CAAC;IAExD,IAAI,OAAO,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,CAAC;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAExC,qEAAqE;IACrE,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAC5C,EAAE,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO;aAC9B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CACT;gBACE,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE;gBAC/B,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,eAAe;aACzC,EACD,gIAAgI,CACjI,CAAC;YACF,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,iBAAiB,CAC5B,IAAI,CAAC,OAAO,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAC1C,MAAM,CAAC,KAAK,EACZ,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxD,CAAC;QACF,MAAM,QAAQ,GAA0B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAClE,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO;SACR,CAAC,CAAC,CAAC;QACJ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5C,MAAM,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,CAAC;IAE5C,wEAAwE;IACxE,IAAI,MAAM,GAAG,sBAAsB,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,cAAc,GAGf,EAAE,CAAC;IACR,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,OAAe,EAAE,EAAE;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC/B,CAAC,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,KAAK,CAAC;YAAE,MAAM;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YACtC,WAAW,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,SAAS,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,KAAK,CAAC;YAAE,MAAM;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ;gBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAChC,cAAc,CAAC,IAAI,CAAC;gBAClB,GAAG,EAAE,MAAM;gBACX,IAAI,EAAE,KAAK,CAAC,IAAI;oBACd,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBAC3D,CAAC,CAAC,IAAI;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,OAAkE,CAAC;QACvE,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,OAAO,GAAG;gBACR,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;aACjE,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;aACvD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC;aAChE,GAAG,CAAC,CAAC,OAAO,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,oBAAoB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACvE,MAAM,CAAC,SAAS,CAAC,IAAI,CACnB,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAC7C,CAAC,OAAO,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CACvD,CACF,CAAC;QACF,wEAAwE;QACxE,6BAA6B;QAC7B,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM;aACnC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC;aACrE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,oBAAoB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAClD,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CACT;gBACE,KAAK;gBACL,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;aAChC,EACD,kCAAkC,CACnC,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CACT;gBACE,KAAK;gBACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;gBAC7B,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;aAC/B,EACD,oCAAoC,CACrC,CAAC;QACJ,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,uEAAuE;IACvE,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC;QAC3C,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,IAAI,IAAI;YAAE,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,CAAC,SAAS,GAAG,CAAC,MAAM,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC9D,OAAO,MAAM,CAAC;AAChB,CAAC;AAQD;;;;;GAKG;AACH,KAAK,UAAU,aAAa,CAC1B,IAA4D,EAC5D,KAAa;IAEb,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7C,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IACzC,CAAC;IACD,IAAI,MAAkC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC;YAC7C,YAAY,EAAE,IAAI,CAAC,WAAW;YAC9B,KAAK;SACN,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC5D,IAAI,iBAAiB,CAAC,MAAM,CAAC,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC1E,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC"}