@opendatalabs/personal-server-ts-core 0.0.1-canary.01ca694 → 0.0.1-canary.052d47e

package/dist/sync/workers/download.d.ts

@@ -1,22 +1,30 @@
 import type { StorageAdapter } from "../../storage/adapters/interface.js";
-import { type FileRecord, type GatewayClient } from "@opendatalabs/vana-sdk/browser";
+import { type DataPointRecord, type GatewayClient } from "@opendatalabs/vana-sdk/browser";
 import type { SyncCursor } from "../cursor.js";
 import type { Logger } from "../../logger/index.js";
-import type { DataStoragePort } from "../../ports/index.js";
-/** Minimal diagnostics hook — keeps core free of lite-specific imports. */
+import type { DataPointFeedPort, DataPointFeedRecord, DataStoragePort, PendingBlobDeletionStore } from "../../ports/index.js";
+import { type ScopeDeletionTracker } from "../scope-deletions.js";
+import { type DownloadRetryMemory } from "../retry-memory.js";
+/**
+ * Minimal diagnostics hook — keeps core free of lite-specific imports.
+ *
+ * Note: the `id` params are the DPv2 data-point id (the stable identifier for a
+ * synced item under the DataPoint model). The param name is kept generic-ish
+ * for the lite diagnostics consumers that predate the file→DataPoint rename.
+ */
 export interface DownloadDiagnosticsHook {
-    onDownloadStart(fileId: string): void;
-    onDownloadEnd(fileId: string, scope: string): void;
-    onDownloadError(fileId: string, detail: string): void;
-    onDecryptStart(fileId: string, scope: string): void;
-    onDecryptEnd(fileId: string, scope: string): void;
-    onDecryptError(fileId: string, scope: string, detail: string): void;
-    onIndexStart(fileId: string, scope: string): void;
-    onIndexEnd(fileId: string, scope: string): void;
-    onIndexError(fileId: string, scope: string, detail: string): void;
-    onManifestBuildStart(fileId: string, scope: string): void;
-    onManifestBuildEnd(fileId: string, scope: string): void;
-    onManifestBuildError(fileId: string, scope: string, detail: string): void;
+    onDownloadStart(id: string): void;
+    onDownloadEnd(id: string, scope: string): void;
+    onDownloadError(id: string, detail: string): void;
+    onDecryptStart(id: string, scope: string): void;
+    onDecryptEnd(id: string, scope: string): void;
+    onDecryptError(id: string, scope: string, detail: string): void;
+    onIndexStart(id: string, scope: string): void;
+    onIndexEnd(id: string, scope: string): void;
+    onIndexError(id: string, scope: string, detail: string): void;
+    onManifestBuildStart(id: string, scope: string): void;
+    onManifestBuildEnd(id: string, scope: string): void;
+    onManifestBuildError(id: string, scope: string, detail: string): void;
     onRepair(scope: string, detail: string): void;
 }
 export interface DownloadWorkerDeps {
@@ -29,29 +37,102 @@ export interface DownloadWorkerDeps {
     logger: Logger;
     /** Optional diagnostics hook — omit to disable instrumentation. */
     diagnostics?: DownloadDiagnosticsHook;
+    /**
+     * Deletion-aware registry listing (`includeDeleted=true`). When present,
+     * tombstoned points reach this replica and their local copies are removed
+     * instead of re-pulled. Falls back to the SDK client listing, which only
+     * sees deletions the gateway volunteers without being asked.
+     */
+    dataPointFeed?: DataPointFeedPort;
+    /**
+     * Read-side deletion memory. Every listed row is recorded here (deleted or
+     * live) and a listing with no further pages marks the memory complete, so
+     * reads can refuse tombstoned scopes without their own gateway lookups.
+     */
+    scopeDeletions?: ScopeDeletionTracker;
+    /**
+     * Exact-key cleanup queue. An unsynced local entry a tombstone covers may
+     * have had its ciphertext uploaded under a version key the registry never
+     * recorded (crashed before registering); reconciling drops the entry and
+     * queues that exact key so the sync cycle's rate-limited pass removes it.
+     */
+    pendingBlobDeletions?: PendingBlobDeletionStore;
+    /**
+     * Called after a downloaded data point is written and indexed locally
+     * (a new local version of `scope`). The derivative compute layer marks
+     * questions that read the scope stale. Best-effort: a throwing hook is
+     * logged and never fails the download.
+     */
+    onDataPointIndexed?: (event: {
+        scope: string;
+        dataPointId: string;
+        version: number;
+        collectedAt: string;
+        /** The record's stamped `$lineage.sources`, when it is a derivative. */
+        lineageSources?: string[];
+    }) => void;
+}
+export interface DeletionReconcileResult {
+    scope: string;
+    deletedAt: string;
+    /** Local versions removed because the tombstone covers them. */
+    removed: number;
+    /** Local unsynced versions ingested after the deletion and kept. */
+    kept: number;
 }
 export interface DownloadResult {
-    fileId: string;
+    dataPointId: string;
     scope: string;
     collectedAt: string;
     path: string;
 }
+export interface DownloadAllOptions {
+    /**
+     * Ignore the stored incremental cursor and reconcile from the beginning of
+     * the owner's data-point registry. This repairs browser caches whose cursor
+     * advanced while local IndexedDB/OPFS lost or failed to index some records.
+     */
+    fullReconcile?: boolean;
+    /**
+     * Cross-cycle failure memory. Records are re-listed every cycle (the cursor
+     * only advances past fully-successful pages), so without this a failing
+     * blob is re-downloaded once per cycle — a 404 forever. The sync manager
+     * owns one instance per boot session.
+     */
+    retryMemory?: DownloadRetryMemory;
+}
+interface LocalMissingBlockRepairSummary {
+    repaired: number;
+    missingEnvelopeEntries: number;
+}
 /**
- * Download and process a single file record from the storage backend:
- * 1. Check dedup: skip if fileId already in local index
- * 2. Download OpenPGP encrypted binary from storage backend
- * 3. Resolve schemaId → scope via Gateway getSchema
+ * Download and process a single DPv2 data point from the storage backend:
+ * 1. Check dedup: skip if dataPointId already in local index
+ * 2. Reconstruct the blob URL from (scope, expectedVersion) — DataPointRecords
+ *    carry no URL, so we rebuild the version-keyed `{scope}/{version}` key the
+ *    upload worker wrote under and resolve it via storageAdapter.urlForKey
+ * 3. Download the OpenPGP encrypted binary from storage
  * 4. Derive scope key from master key → hex-encode as OpenPGP password
+ *    (scope comes straight off the record — no schema lookup needed)
  * 5. Decrypt with OpenPGP password-based decryption → plaintext JSON
  * 6. Parse as DataFileEnvelope (validate)
- * 7. Write to local filesystem via hierarchy manager
- * 8. Insert into local index (with fileId)
+ * 7. Write to local filesystem + bounded block sidecars
+ * 8. Insert into local index (with dataPointId + version)
+ */
+export declare function downloadOne(deps: DownloadWorkerDeps, record: DataPointRecord): Promise<DownloadResult | null>;
+/**
+ * Poll Gateway for the owner's data points since the stored cursor, download
+ * each, and advance the cursor only when the page fully succeeds.
  */
-export declare function downloadOne(deps: DownloadWorkerDeps, record: FileRecord): Promise<DownloadResult | null>;
+export declare function downloadAll(deps: DownloadWorkerDeps, options?: DownloadAllOptions): Promise<DownloadResult[]>;
 /**
- * Poll Gateway for new file records since lastProcessedTimestamp,
- * download each, and advance the cursor only when the page fully succeeds.
+ * Apply a gateway tombstone to the local index: remove every local version
+ * the tombstone covers (synced rows at or below its version, and unsynced
+ * rows ingested without knowledge of it), keep unsynced rows ingested on top
+ * of it (a legitimate re-add that the upload worker registers after the
+ * tombstone version). See `isEntryCoveredByTombstone`.
  */
-export declare function downloadAll(deps: DownloadWorkerDeps): Promise<DownloadResult[]>;
-export declare function repairLocalMissingBlockSidecars(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "diagnostics">): Promise<number>;
+export declare function reconcileDeletedDataPoint(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "pendingBlobDeletions">, record: Pick<DataPointFeedRecord, "id" | "scope" | "expectedVersion">, deletedAt: string): Promise<DeletionReconcileResult>;
+export declare function repairLocalMissingBlockSidecars(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "diagnostics">): Promise<LocalMissingBlockRepairSummary>;
+export {};
 //# sourceMappingURL=download.d.ts.map

package/dist/sync/workers/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAIL,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAS5D,2EAA2E;AAC3E,MAAM,WAAW,uBAAuB;IACtC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACpD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACpE,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClE,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1D,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,UAAU,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,WAAW,CAAC,EAAE,uBAAuB,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AASD;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAuKhC;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,cAAc,EAAE,CAAC,CA0F3B;AAED,wBAAsB,+BAA+B,CACnD,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,aAAa,CAAC,GACnE,OAAO,CAAC,MAAM,CAAC,CA4CjB"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EAEf,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,EAAoB,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGhF;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/C,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAChE,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9D,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACpD,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,UAAU,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,WAAW,CAAC,EAAE,uBAAuB,CAAC;IACtC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC;;;;OAIG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,wBAAwB,CAAC;IAChD;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,wEAAwE;QACxE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,KAAK,IAAI,CAAC;CACZ;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACnC;AAYD,UAAU,8BAA8B;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAYD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAoNhC;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,cAAc,EAAE,CAAC,CAuK3B;AAED;;;;;;GAMG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC,EAC7E,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,GAAG,OAAO,GAAG,iBAAiB,CAAC,EACrE,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,uBAAuB,CAAC,CAsDlC;AAED,wBAAsB,+BAA+B,CACnD,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,aAAa,CAAC,GACnE,OAAO,CAAC,8BAA8B,CAAC,CAyDzC"}