@opendatalabs/personal-server-ts-core 0.0.1-canary.00328bf

package/dist/sync/workers/download.d.ts

@@ -0,0 +1,159 @@
+import type { StorageAdapter } from "../../storage/adapters/interface.js";
+import { type DataPointRecord, type GatewayClient } from "@opendatalabs/vana-sdk/browser";
+import type { SyncCursor } from "../cursor.js";
+import type { Logger } from "../../logger/index.js";
+import type { DataPointFeedPort, DataPointFeedRecord, DataStoragePort, PendingBlobDeletionStore } from "../../ports/index.js";
+import { type ScopeDeletionTracker } from "../scope-deletions.js";
+import { type DownloadRetryMemory } from "../retry-memory.js";
+import { type PdppImporter } from "../pdpp-import.js";
+/**
+ * Minimal diagnostics hook — keeps core free of lite-specific imports.
+ *
+ * Note: the `id` params are the DPv2 data-point id (the stable identifier for a
+ * synced item under the DataPoint model). The param name is kept generic-ish
+ * for the lite diagnostics consumers that predate the file→DataPoint rename.
+ */
+export interface DownloadDiagnosticsHook {
+    onDownloadStart(id: string): void;
+    onDownloadEnd(id: string, scope: string): void;
+    onDownloadError(id: string, detail: string): void;
+    onDecryptStart(id: string, scope: string): void;
+    onDecryptEnd(id: string, scope: string): void;
+    onDecryptError(id: string, scope: string, detail: string): void;
+    onIndexStart(id: string, scope: string): void;
+    onIndexEnd(id: string, scope: string): void;
+    onIndexError(id: string, scope: string, detail: string): void;
+    onManifestBuildStart(id: string, scope: string): void;
+    onManifestBuildEnd(id: string, scope: string): void;
+    onManifestBuildError(id: string, scope: string, detail: string): void;
+    onRepair(scope: string, detail: string): void;
+}
+export interface DownloadWorkerDeps {
+    storage: DataStoragePort;
+    storageAdapter: StorageAdapter;
+    gateway: GatewayClient;
+    cursor: SyncCursor;
+    masterKey: Uint8Array;
+    serverOwner: string;
+    logger: Logger;
+    /** Optional diagnostics hook — omit to disable instrumentation. */
+    diagnostics?: DownloadDiagnosticsHook;
+    /**
+     * Deletion-aware registry listing (`includeDeleted=true`). When present,
+     * tombstoned points reach this replica and their local copies are removed
+     * instead of re-pulled. Falls back to the SDK client listing, which only
+     * sees deletions the gateway volunteers without being asked.
+     */
+    dataPointFeed?: DataPointFeedPort;
+    /**
+     * Read-side deletion memory. Every listed row is recorded here (deleted or
+     * live) and a listing with no further pages marks the memory complete, so
+     * reads can refuse tombstoned scopes without their own gateway lookups.
+     */
+    scopeDeletions?: ScopeDeletionTracker;
+    /**
+     * Exact-key cleanup queue. An unsynced local entry a tombstone covers may
+     * have had its ciphertext uploaded under a version key the registry never
+     * recorded (crashed before registering); reconciling drops the entry and
+     * queues that exact key so the sync cycle's rate-limited pass removes it.
+     */
+    pendingBlobDeletions?: PendingBlobDeletionStore;
+    /**
+     * Called after a downloaded data point is written and indexed locally
+     * (a new local version of `scope`). The derivative compute layer marks
+     * questions that read the scope stale. Best-effort: a throwing hook is
+     * logged and never fails the download.
+     */
+    onDataPointIndexed?: (event: {
+        scope: string;
+        dataPointId: string;
+        version: number;
+        collectedAt: string;
+        /** The record's stamped `$lineage.sources`, when it is a derivative. */
+        lineageSources?: string[];
+    }) => void;
+    /**
+     * Import a decrypted envelope into the PDPP record store, when the
+     * deployment mounted one.
+     *
+     * Runs AFTER the legacy write and index above, and never in place of them:
+     * PDPP is an additional read surface over the same synced data, not a
+     * replacement for the storage every existing reader depends on. A failing
+     * import is logged and swallowed for the same reason — the data point is
+     * already durably stored and indexed by the time this runs, so throwing
+     * would block the sync cursor and stall the legacy path over a defect in
+     * the newer one.
+     */
+    pdppImporter?: PdppImporter;
+}
+export interface DeletionReconcileResult {
+    scope: string;
+    deletedAt: string;
+    /** Local versions removed because the tombstone covers them. */
+    removed: number;
+    /** Local unsynced versions ingested after the deletion and kept. */
+    kept: number;
+}
+export interface DownloadResult {
+    dataPointId: string;
+    scope: string;
+    collectedAt: string;
+    path: string;
+}
+export interface DownloadAllOptions {
+    /**
+     * Ignore the stored incremental cursor and reconcile from the beginning of
+     * the owner's data-point registry. This repairs browser caches whose cursor
+     * advanced while local IndexedDB/OPFS lost or failed to index some records.
+     */
+    fullReconcile?: boolean;
+    /**
+     * Cross-cycle failure memory. Records are re-listed every cycle (the cursor
+     * only advances past fully-successful pages), so without this a failing
+     * blob is re-downloaded once per cycle — a 404 forever. The sync manager
+     * owns one instance per boot session.
+     */
+    retryMemory?: DownloadRetryMemory;
+}
+export type DownloadScopesOptions = Pick<DownloadAllOptions, "retryMemory">;
+/**
+ * Hydrate exact owner scopes without listing the registry or changing the
+ * incremental cursor. This is the cold-start fast path; downloadAll remains
+ * responsible for the complete registry reconciliation.
+ */
+export declare function downloadScopes(deps: DownloadWorkerDeps, scopes: string[], options?: DownloadScopesOptions): Promise<DownloadResult[]>;
+interface LocalMissingBlockRepairSummary {
+    repaired: number;
+    missingEnvelopeEntries: number;
+}
+/**
+ * Download and process a single DPv2 data point from the storage backend:
+ * 1. Check dedup: skip if dataPointId already in local index
+ * 2. Reconstruct the blob URL from (scope, expectedVersion) — DataPointRecords
+ *    carry no URL, so we rebuild the version-keyed `{scope}/{version}` key the
+ *    upload worker wrote under and resolve it via storageAdapter.urlForKey
+ * 3. Download the OpenPGP encrypted binary from storage
+ * 4. Derive scope key from master key → hex-encode as OpenPGP password
+ *    (scope comes straight off the record — no schema lookup needed)
+ * 5. Decrypt with OpenPGP password-based decryption → plaintext JSON
+ * 6. Parse as DataFileEnvelope (validate)
+ * 7. Write to local filesystem + bounded block sidecars
+ * 8. Insert into local index (with dataPointId + version)
+ */
+export declare function downloadOne(deps: DownloadWorkerDeps, record: DataPointRecord): Promise<DownloadResult | null>;
+/**
+ * Poll Gateway for the owner's data points since the stored cursor, download
+ * each, and advance the cursor only when the page fully succeeds.
+ */
+export declare function downloadAll(deps: DownloadWorkerDeps, options?: DownloadAllOptions): Promise<DownloadResult[]>;
+/**
+ * Apply a gateway tombstone to the local index: remove every local version
+ * the tombstone covers (synced rows at or below its version, and unsynced
+ * rows ingested without knowledge of it), keep unsynced rows ingested on top
+ * of it (a legitimate re-add that the upload worker registers after the
+ * tombstone version). See `isEntryCoveredByTombstone`.
+ */
+export declare function reconcileDeletedDataPoint(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "pendingBlobDeletions">, record: Pick<DataPointFeedRecord, "id" | "scope" | "expectedVersion">, deletedAt: string): Promise<DeletionReconcileResult>;
+export declare function repairLocalMissingBlockSidecars(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "diagnostics">): Promise<LocalMissingBlockRepairSummary>;
+export {};
+//# sourceMappingURL=download.d.ts.map

package/dist/sync/workers/download.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EAEf,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,EAAoB,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEhF,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAE5E;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/C,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAChE,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9D,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACpD,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,UAAU,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,WAAW,CAAC,EAAE,uBAAuB,CAAC;IACtC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC;;;;OAIG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,wBAAwB,CAAC;IAChD;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,wEAAwE;QACxE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX;;;;;;;;;;;OAWG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACnC;AAED,MAAM,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,EAAE,aAAa,CAAC,CAAC;AAE5E;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,cAAc,EAAE,CAAC,CAkE3B;AAYD,UAAU,8BAA8B;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAYD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAsOhC;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,cAAc,EAAE,CAAC,CAuK3B;AAED;;;;;;GAMG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC,EAC7E,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,GAAG,OAAO,GAAG,iBAAiB,CAAC,EACrE,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,uBAAuB,CAAC,CAsDlC;AAED,wBAAsB,+BAA+B,CACnD,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,aAAa,CAAC,GACnE,OAAO,CAAC,8BAA8B,CAAC,CAyDzC"}