@opendatalabs/personal-server-ts-core 1.7.0 → 1.9.0

package/dist/sync/tombstone.js

@@ -0,0 +1,28 @@
+import { keccak256, stringToHex } from "viem";
+/**
+ * Durable-deletion tombstone commitments.
+ *
+ * A deletion is an owner-signed DPv2 AddData for version `current + 1` whose
+ * dataHash / metadataHash are these well-known constants. The gateway, the
+ * SDK and every personal server derive the SAME bytes from the same UTF-8
+ * labels, so a tombstone is recognisable on-chain and in every feed without
+ * any out-of-band flag. Do not change the labels: they are the cross-repo
+ * contract.
+ *
+ * TODO(vana-sdk > 3.14.0): the SDK now exports TOMBSTONE_DATA_HASH,
+ * TOMBSTONE_METADATA_HASH, isTombstoneHashes(), isDataPointTombstone() and
+ * deriveDataPointId() with these exact values. Switch to those exports and
+ * drop the local copies once the PS pins a release that ships them.
+ */
+export const TOMBSTONE_DATA_HASH_LABEL = "vana.data-point.tombstone.v1";
+export const TOMBSTONE_METADATA_HASH_LABEL = "vana.data-point.tombstone.metadata.v1";
+/** keccak256(utf8("vana.data-point.tombstone.v1")) */
+export const TOMBSTONE_DATA_HASH = keccak256(stringToHex(TOMBSTONE_DATA_HASH_LABEL));
+/** keccak256(utf8("vana.data-point.tombstone.metadata.v1")) */
+export const TOMBSTONE_METADATA_HASH = keccak256(stringToHex(TOMBSTONE_METADATA_HASH_LABEL));
+/** True when a data-point record's commitments are the tombstone constants. */
+export function isTombstoneRecord(record) {
+    return (record.dataHash.toLowerCase() === TOMBSTONE_DATA_HASH &&
+        record.metadataHash.toLowerCase() === TOMBSTONE_METADATA_HASH);
+}
+//# sourceMappingURL=tombstone.js.map

package/dist/sync/tombstone.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tombstone.js","sourceRoot":"","sources":["../../src/sync/tombstone.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AAE9C;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,8BAA8B,CAAC;AACxE,MAAM,CAAC,MAAM,6BAA6B,GACxC,uCAAuC,CAAC;AAE1C,sDAAsD;AACtD,MAAM,CAAC,MAAM,mBAAmB,GAAkB,SAAS,CACzD,WAAW,CAAC,yBAAyB,CAAC,CACvC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,CAAC,MAAM,uBAAuB,GAAkB,SAAS,CAC7D,WAAW,CAAC,6BAA6B,CAAC,CAC3C,CAAC;AAEF,+EAA+E;AAC/E,MAAM,UAAU,iBAAiB,CAAC,MAGjC;IACC,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,mBAAmB;QACrD,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,uBAAuB,CAC9D,CAAC;AACJ,CAAC"}

package/dist/sync/workers/delete.d.ts

@@ -1,42 +1,163 @@
-import type { StorageAdapter } from "../../storage/adapters/interface.js";
-import type { GatewayClient } from "@opendatalabs/vana-sdk/browser";
-import type { ServerSigner } from "../../signing/signer.js";
 import type { Logger } from "../../logger/index.js";
-import type { DataStoragePort } from "../../ports/index.js";
-export interface DeleteWorkerDeps {
+import type { DataPointFeedPort, DataStoragePort, DeleteDataPort, PendingBlobDeletion, PendingBlobDeletionStore } from "../../ports/index.js";
+import { type ScopeDeletionTracker } from "../scope-deletions.js";
+/**
+ * Exact blob deletes sent per pass. vana-storage allows 30 DELETE requests
+ * per owner per 60 seconds. A DELETE request's own pass and the next sync
+ * cycle's retry pass can land inside one window, so each pass sends at most
+ * half the budget; whatever does not fit is queued and drained on later
+ * cycles, and a 429 simply re-queues its key.
+ */
+export declare const BLOB_DELETE_BATCH_SIZE = 15;
+export interface DeleteScopeDeps {
     storage: DataStoragePort;
-    storageAdapter: StorageAdapter;
-    gateway: GatewayClient;
-    signer: ServerSigner;
-    serverOwner: string;
+    /** Data owner; absent only on an unconfigured host (no key material). */
+    serverOwner?: string;
+    /**
+     * Remote deletion (gateway tombstone + storage blobs). Absent when the host
+     * runs without sync (no owner key material): deletion is then local-only
+     * and reported as such, never silently "durable".
+     */
+    deleteData?: DeleteDataPort | null;
+    /** Exact-key retry markers for blob deletes that did not finish. */
+    pendingBlobDeletions?: PendingBlobDeletionStore;
+    /**
+     * Read-side deletion memory. A landed tombstone is recorded here so this
+     * replica's reads answer 410 immediately, before any sync cycle.
+     */
+    scopeDeletions?: ScopeDeletionTracker;
+    /**
+     * Deletion-aware registry lookup, re-read right after the tombstone attempt
+     * (a registration racing a "not-registered" answer) and by the retry to
+     * expand a marker whose tombstone version was not known.
+     */
+    dataPointFeed?: DataPointFeedPort;
     logger: Logger;
+    now?: () => Date;
+}
+export type DeleteStepStatus = "ok" | "skipped" | "failed" | "deferred";
+export interface DeleteScopeStep {
+    status: DeleteStepStatus;
+    /** Why the step was skipped (e.g. "not-registered", "sync-disabled"). */
+    reason?: string;
+    /** Error message when status is "failed". */
+    error?: string;
 }
-export interface DeleteScopeRemoteResult {
+export interface DeleteScopeResult {
     scope: string;
-    /** Synced versions de-registered at the gateway. */
-    filesDeregistered: number;
-    /** Ciphertext blobs hard-deleted from storage. */
-    blobsDeleted: number;
-    errors: Array<{
-        fileId: string | null;
-        message: string;
-    }>;
+    /** DPv2 identity: keccak256(abi.encode(owner, scope)); null without an owner. */
+    dataPointId: string | null;
+    /**
+     * True once the gateway holds the tombstone (or never held the point at
+     * all): sync can no longer resurrect the scope on any replica. False when
+     * only the local copy was removed.
+     */
+    durable: boolean;
+    steps: {
+        gateway: DeleteScopeStep & {
+            /** Tombstone version at the gateway; null when it did not report one. */
+            version?: string | null;
+            deletedAt?: string | null;
+        };
+        /**
+         * Storage step. "ok": every blob key the tombstone covers is gone.
+         * "deferred": some keys are queued for later passes (rate-limit batch,
+         * or a tombstone version the gateway did not report). "failed": at
+         * least one delete errored; the failed keys are queued too.
+         */
+        storage: DeleteScopeStep & {
+            blobsDeleted?: number;
+            blobsMissing?: number;
+            blobsPending?: number;
+        };
+        local: DeleteScopeStep & {
+            deletedCount?: number;
+        };
+    };
+    /**
+     * True when the tombstone landed but blob deletion did not finish: exact
+     * retry markers were recorded and later sync cycles finish them.
+     */
+    pendingBlobDeletion: boolean;
+}
+export interface BlobDeletionPlan {
+    /**
+     * Exact keys of covered local entries that lie outside the registry range:
+     * an upload that landed under a key the registry never saw (crashed before
+     * registering, or rebased later) is still deleted data. Bounded by the
+     * local index size.
+     */
+    keys: string[];
+    /**
+     * Every registry version up to and including the tombstone's, as inclusive
+     * bounds. Versions are a dense, strictly increasing integer sequence per
+     * scope and an absent key answers 404, so the range is exact and complete
+     * across replicas without listing storage. Kept as bounds, never
+     * materialised: a large tombstone version costs nothing until its keys are
+     * actually sent, a batch at a time. Null without a tombstone version.
+     */
+    range: {
+        from: string;
+        to: string;
+    } | null;
 }
 /**
- * Propagate a scope deletion to the authoritative remote stores.
+ * The storage keys a tombstone covers for `scope`. A re-add registered after
+ * the tombstone lives under a higher version key, so it is never in this set
+ * and survives by construction. With no tombstone version (never registered,
+ * or the gateway did not report one) only the local entries' keys can be
+ * enumerated.
+ */
+export declare function planBlobDeletions(storage: Pick<DataStoragePort, "listVersions">, scope: string, tombstoneVersionValue: string | null): BlobDeletionPlan;
+/**
+ * Durable scope deletion, in the only order that cannot resurrect data:
+ *
+ *   1. gateway tombstone  (owner-signed AddData at current + 1; the fact)
+ *   2. storage blobs      (exact keys of every version the tombstone covers)
+ *   3. local copy         (index rows + files + sidecars)
  *
- * NOT SUPPORTED on the DPv2 (DataPoint) gateway: there is no data-point
- * de-registration / deletion endpoint, and the DataPointRecord listing carries
- * no soft-delete (`deletedAt`) flag. Under the legacy file model this worker
- * de-registered each file then hard-deleted its blob; that path is gone with
- * `gateway.getFile` / `gateway.deleteFile` / `signFileDeletion`.
+ * The local delete never runs before the gateway has acknowledged: until the
+ * registry says "deleted", the next sync cycle would re-pull the point from
+ * the still-live row. Step 2 deletes exact per-version keys, never the scope
+ * prefix, so a re-add another replica lands at any time keeps its ciphertext
+ * (it is registered above the tombstone version and stored under that key).
+ * Keys that could not be deleted in this pass are queued as exact retry
+ * markers; step 3 does not wait for them because the tombstone already makes
+ * the deletion stick.
  *
- * Deleting the ciphertext blob alone would be unsafe: the data point would
- * stay listed, so any server syncing it would 404 on download and wedge its
- * sync cursor. So this is a no-op that logs the limitation. Local deletion
- * (index rows + on-disk blobs) still happens via the storage port's
- * `deleteScope`, independent of this worker — only the *remote* copy is left
- * in place until DPv2 grows a deletion API.
+ * Not serialised here: the sync manager runs this under the same lock as its
+ * upload/download cycles so a delete cannot interleave with an upload of the
+ * same scope on this replica (see engine/sync-manager.ts). Uploads racing on
+ * another replica are handled by the upload worker's deletion guard and the
+ * download worker's reconcile.
+ */
+export declare function deleteScope(deps: DeleteScopeDeps, scope: string): Promise<DeleteScopeResult>;
+export interface RetryPendingBlobDeletionsResult {
+    /** Exact keys finished this pass (deleted, or already absent). */
+    completed: PendingBlobDeletion[];
+    /** Unexpanded markers dropped because the scope is live again. */
+    superseded: string[];
+    failed: Array<{
+        scope: string;
+        version: string | null;
+        error: string;
+    }>;
+    /** Markers still queued after this pass (a range counts once). */
+    remaining: number;
+}
+/**
+ * Finish blob deletions whose tombstone landed but whose storage deletes did
+ * not: one batch of exact keys per sync cycle, taken from exact-key markers
+ * first and then from the head of range markers (which advance in place),
+ * each key retried until storage acknowledges (2xx or 404). Unexpanded
+ * markers (the gateway did not report a tombstone version at delete time)
+ * are expanded from the registry first; if the registry shows the scope live
+ * again the old tombstone's version is unknowable, so the marker is dropped
+ * and the pre-tombstone ciphertext stays (owner-decryptable only) until the
+ * next delete of that scope.
  */
-export declare function deleteScopeRemote(deps: DeleteWorkerDeps, scope: string): Promise<DeleteScopeRemoteResult>;
+export declare function retryPendingBlobDeletions(deps: Pick<DeleteScopeDeps, "deleteData" | "pendingBlobDeletions" | "dataPointFeed" | "serverOwner" | "logger"> & {
+    /** Local index, for expanding unexpanded markers into exact keys. */
+    storage?: Pick<DataStoragePort, "listVersions">;
+}): Promise<RetryPendingBlobDeletionsResult>;
 //# sourceMappingURL=delete.d.ts.map

package/dist/sync/workers/delete.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/delete.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,uBAAuB,CAAC,CA2BlC"}
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/delete.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EACV,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAEzC,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACnC,oEAAoE;IACpE,oBAAoB,CAAC,EAAE,wBAAwB,CAAC;IAChD;;;OAGG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC;;;;OAIG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,MAAM,MAAM,gBAAgB,GAAG,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,CAAC;AAExE,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,gBAAgB,CAAC;IACzB,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE;QACL,OAAO,EAAE,eAAe,GAAG;YACzB,yEAAyE;YACzE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;YACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;SAC3B,CAAC;QACF;;;;;WAKG;QACH,OAAO,EAAE,eAAe,GAAG;YACzB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,YAAY,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,KAAK,EAAE,eAAe,GAAG;YAAE,YAAY,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAAC;IACF;;;OAGG;IACH,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;;;;;;OAOG;IACH,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,cAAc,CAAC,EAC9C,KAAK,EAAE,MAAM,EACb,qBAAqB,EAAE,MAAM,GAAG,IAAI,GACnC,gBAAgB,CAqBlB;AAyCD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,eAAe,EACrB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,iBAAiB,CAAC,CAiJ5B;AA2FD,MAAM,WAAW,+BAA+B;IAC9C,kEAAkE;IAClE,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,kEAAkE;IAClE,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxE,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,IAAI,CACR,eAAe,EACb,YAAY,GACZ,sBAAsB,GACtB,eAAe,GACf,aAAa,GACb,QAAQ,CACX,GAAG;IACF,qEAAqE;IACrE,OAAO,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;CACjD,GACA,OAAO,CAAC,+BAA+B,CAAC,CAyJ1C"}