@opendatalabs/personal-server-ts-core 0.2.0 → 1.0.1

package/dist/sync/workers/download.js

@@ -1,90 +1,432 @@
-import { writeDataFile } from "../../storage/hierarchy/index.js";
-import { deriveScopeKey } from "../../keys/derive.js";
-import { decryptWithPassword } from "../../storage/encryption/index.js";
-import { DataFileEnvelopeSchema } from "../../schemas/data-file.js";
+import { DataFileEnvelopeSchema, decryptWithPassword, deriveScopeKey, } from "@opendatalabs/vana-sdk/browser";
+import { buildDataBlocksAsync } from "../../storage/blocks/build.js";
+import { classifySyncFailure, inferPayloadKind, } from "../issues.js";
+import { downloadRetryKey } from "../retry-memory.js";
 /**
- * Download and process a single file record from the storage backend:
- * 1. Check dedup: skip if fileId already in local index
- * 2. Download OpenPGP encrypted binary from storage backend
- * 3. Resolve schemaId → scope via Gateway getSchema
+ * Download and process a single DPv2 data point from the storage backend:
+ * 1. Check dedup: skip if dataPointId already in local index
+ * 2. Reconstruct the blob URL from (scope, expectedVersion) — DataPointRecords
+ *    carry no URL, so we rebuild the version-keyed `{scope}/{version}` key the
+ *    upload worker wrote under and resolve it via storageAdapter.urlForKey
+ * 3. Download the OpenPGP encrypted binary from storage
  * 4. Derive scope key from master key → hex-encode as OpenPGP password
+ *    (scope comes straight off the record — no schema lookup needed)
  * 5. Decrypt with OpenPGP password-based decryption → plaintext JSON
  * 6. Parse as DataFileEnvelope (validate)
- * 7. Write to local filesystem via hierarchy manager
- * 8. Insert into local index (with fileId)
+ * 7. Write to local filesystem + bounded block sidecars
+ * 8. Insert into local index (with dataPointId + version)
  */
 export async function downloadOne(deps, record) {
-    const { indexManager, hierarchyOptions, storageAdapter, gateway, masterKey, logger, } = deps;
-    // 1. Check dedup: skip if fileId already in local index
-    const existing = indexManager.findByFileId(record.fileId);
+    const { storage, storageAdapter, masterKey, logger, diagnostics } = deps;
+    const ctx = { dataPointId: record.id, schemaId: null };
+    // 1. Check dedup: skip if dataPointId already in local index
+    const existing = storage.findByDataPointId(record.id);
     if (existing) {
-        logger.debug({ fileId: record.fileId }, "File already in index, skipping");
-        return null;
-    }
-    // 2. Download OpenPGP encrypted binary from storage backend
-    const encrypted = await storageAdapter.download(record.url);
-    // 3. Resolve schemaId → scope via Gateway getSchema
-    const schema = await gateway.getSchema(record.schemaId);
-    if (!schema) {
-        throw new Error(`No schema found for schemaId: ${record.schemaId}`);
-    }
-    // 4. Derive scope key → hex-encode as OpenPGP password
-    const scopeKey = deriveScopeKey(masterKey, schema.scope);
-    const scopeKeyHex = Buffer.from(scopeKey).toString("hex");
+        const repairResult = await repairMissingBlockSidecars(storage, logger, { dataPointId: record.id, schemaId: existing.schemaId }, existing, diagnostics);
+        if (repairResult === "missing-envelope") {
+            logger.warn({
+                dataPointId: record.id,
+                scope: existing.scope,
+                collectedAt: existing.collectedAt,
+            }, "Local index entry was missing its envelope; re-downloading data point");
+        }
+        else {
+            logger.debug({ dataPointId: record.id }, "Data point already in index, skipping");
+            return null;
+        }
+    }
+    // 2. Reconstruct the version-keyed storage URL.
+    const storageKey = `${record.scope}/${record.expectedVersion}`;
+    const url = storageAdapter.urlForKey(storageKey);
+    // 3. Download OpenPGP encrypted binary from storage backend. A download
+    // failure throws and blocks the cursor (correct: a transient error retries
+    // next cycle rather than risk a silent gap). Stamp the stage so the
+    // classifier can tell a deterministic 404 (quarantine, never retry) from a
+    // transient storage error — without it the failure infers stage "unknown"
+    // and every 404 is retried forever.
+    diagnostics?.onDownloadStart(record.id);
+    let encrypted;
+    try {
+        encrypted = await storageAdapter.download(url);
+    }
+    catch (err) {
+        const detail = err.message;
+        diagnostics?.onDownloadError(record.id, detail);
+        throw withSyncFailureMetadata(err, {
+            stage: "download",
+            scope: record.scope,
+        });
+    }
+    diagnostics?.onDownloadEnd(record.id, record.scope);
+    // 4. Derive scope key → hex-encode as OpenPGP password (scope off the record)
+    const scopeKey = deriveScopeKey(masterKey, record.scope);
+    const scopeKeyHex = uint8ToHex(scopeKey);
     // 5. Decrypt with OpenPGP password-based decryption
-    const plaintext = await decryptWithPassword(encrypted, scopeKeyHex);
+    diagnostics?.onDecryptStart(record.id, record.scope);
+    let plaintext;
+    try {
+        plaintext = await decryptWithPassword(encrypted, scopeKeyHex);
+    }
+    catch (err) {
+        const payloadKind = inferPayloadKind(encrypted);
+        const encryptedSizeBytes = encrypted.byteLength;
+        const detail = [
+            err.message,
+            `payloadKind=${payloadKind}`,
+            `encryptedSizeBytes=${encryptedSizeBytes}`,
+        ].join("; ");
+        diagnostics?.onDecryptError(record.id, record.scope, detail);
+        throw withSyncFailureMetadata(err, {
+            scope: record.scope,
+            payloadKind,
+            encryptedSizeBytes,
+        });
+    }
     // 6. Parse as DataFileEnvelope (validate)
-    const raw = JSON.parse(new TextDecoder().decode(plaintext));
-    const envelope = DataFileEnvelopeSchema.parse(raw);
-    // 7. Write to local filesystem via hierarchy manager
-    const { relativePath, sizeBytes } = await writeDataFile(hierarchyOptions, envelope);
-    // 8. Insert into local index (with fileId)
-    indexManager.insert({
-        fileId: record.fileId,
-        path: relativePath,
+    let envelope;
+    try {
+        const raw = JSON.parse(new TextDecoder().decode(plaintext));
+        envelope = DataFileEnvelopeSchema.parse(raw);
+    }
+    catch (err) {
+        const detail = err.message;
+        diagnostics?.onDecryptError(record.id, record.scope, detail);
+        throw err;
+    }
+    ctx.schemaId = envelope.schemaId ?? null;
+    diagnostics?.onDecryptEnd(record.id, envelope.scope);
+    const existingByVersion = storage.findEntry({
         scope: envelope.scope,
-        collectedAt: envelope.collectedAt,
-        sizeBytes,
+        at: envelope.collectedAt,
     });
-    logger.info({ fileId: record.fileId, scope: envelope.scope, path: relativePath }, "Downloaded and indexed file");
+    if (existingByVersion !== undefined &&
+        existingByVersion.collectedAt === envelope.collectedAt) {
+        if (existingByVersion.dataPointId !== record.id) {
+            await storage.updateDataPointId(existingByVersion.path, record.id);
+        }
+        const repairResult = await repairMissingBlockSidecars(storage, logger, {
+            dataPointId: record.id,
+            schemaId: existingByVersion.schemaId ?? ctx.schemaId,
+        }, existingByVersion, diagnostics);
+        if (repairResult !== "missing-envelope") {
+            logger.debug({
+                dataPointId: record.id,
+                scope: envelope.scope,
+                collectedAt: envelope.collectedAt,
+            }, "Data version already exists locally, skipping");
+            return null;
+        }
+        logger.warn({
+            dataPointId: record.id,
+            scope: envelope.scope,
+            collectedAt: envelope.collectedAt,
+        }, "Local version entry was missing its envelope; rewriting downloaded file");
+    }
+    // 7. Write to local storage + bounded block sidecars
+    diagnostics?.onIndexStart(record.id, envelope.scope);
+    let relativePath;
+    let sizeBytes;
+    try {
+        ({ relativePath, sizeBytes } = await storage.writeEnvelope(envelope));
+        await writeBlockSidecars(storage, envelope, logger, ctx, relativePath, diagnostics);
+    }
+    catch (err) {
+        const detail = err.message;
+        diagnostics?.onIndexError(record.id, envelope.scope, detail);
+        throw err;
+    }
+    // 8. Insert into local index (with dataPointId + version)
+    try {
+        await storage.insertEntry({
+            fileId: null,
+            schemaId: ctx.schemaId,
+            path: relativePath,
+            scope: envelope.scope,
+            collectedAt: envelope.collectedAt,
+            sizeBytes,
+            version: Number(record.expectedVersion),
+            dataPointId: record.id,
+        });
+    }
+    catch (err) {
+        const detail = err.message;
+        diagnostics?.onIndexError(record.id, envelope.scope, detail);
+        throw err;
+    }
+    diagnostics?.onIndexEnd(record.id, envelope.scope);
+    logger.info({ dataPointId: record.id, scope: envelope.scope, path: relativePath }, "Downloaded and indexed data point");
     return {
-        fileId: record.fileId,
+        dataPointId: record.id,
         scope: envelope.scope,
         collectedAt: envelope.collectedAt,
         path: relativePath,
     };
 }
 /**
- * Poll Gateway for new file records since lastProcessedTimestamp,
- * download each, and advance the cursor.
+ * Poll Gateway for the owner's data points since the stored cursor, download
+ * each, and advance the cursor only when the page fully succeeds.
  */
-export async function downloadAll(deps) {
+export async function downloadAll(deps, options = {}) {
     const { gateway, cursor, serverOwner, logger } = deps;
-    // 1. Read cursor
-    const lastProcessedTimestamp = await cursor.read();
-    // 2. Poll gateway for new file records
-    const { files, cursor: nextCursor } = await gateway.listFilesSince(serverOwner, lastProcessedTimestamp);
+    const syncRunId = createSyncRunId();
+    const repairSummary = await repairLocalMissingBlockSidecars(deps);
+    // 1. Read cursor (opaque pagination cursor persisted across runs)
+    const lastCursor = options.fullReconcile || repairSummary.missingEnvelopeEntries > 0
+        ? null
+        : await cursor.read();
+    if (options.fullReconcile) {
+        logger.info("Running full registry reconciliation to repair any missing local scope records");
+    }
+    if (repairSummary.missingEnvelopeEntries > 0) {
+        logger.warn({ missingEnvelopeEntries: repairSummary.missingEnvelopeEntries }, "Resetting this sync listing to repair stale local index entries");
+    }
+    if (options.fullReconcile || repairSummary.missingEnvelopeEntries > 0) {
+        // An EXPLICIT listing restart (full reconcile, or the repair path above)
+        // exists to re-fetch — give transient failures a fresh attempt budget so
+        // it can; 404s stay dead. Note `lastCursor === null` alone is NOT a
+        // reset signal: single-page listings never advance the cursor, so null
+        // is their steady state and clearing on it would erase the backoff
+        // memory every cycle.
+        options.retryMemory?.onListingReset();
+    }
+    // 2. Poll gateway for the owner's data points.
+    const { dataPoints, cursor: nextCursor } = await gateway.listDataPointsByOwner(serverOwner, lastCursor);
     const results = [];
-    // 3. Process each file record
-    for (const file of files) {
+    let failed = false;
+    // 3. Process each data point record
+    for (const dataPoint of dataPoints) {
+        const retryKey = downloadRetryKey(dataPoint);
+        const decision = options.retryMemory?.decide(retryKey) ?? "attempt";
+        if (decision === "give-up") {
+            // Non-retryable failure (e.g. blob 404) or exhausted transient
+            // retries — skip without blocking the cursor so one dead record
+            // can't wedge the rest of the listing. A new registered version
+            // (different retry key) starts fresh.
+            logger.debug({ dataPointId: dataPoint.id, scope: dataPoint.scope }, "Skipping data point after exhausted download retries");
+            continue;
+        }
+        if (decision === "backoff") {
+            // Waiting out the backoff window. Block the cursor so the record is
+            // still listed when the window expires.
+            failed = true;
+            continue;
+        }
         try {
-            const result = await downloadOne(deps, file);
+            const result = await downloadOne(deps, dataPoint);
+            options.retryMemory?.recordSuccess(retryKey);
             if (result) {
                 results.push(result);
             }
         }
         catch (err) {
+            const metadata = getSyncFailureMetadata(err);
+            const classified = classifySyncFailure({
+                error: err,
+                fileId: dataPoint.id,
+                syncRunId,
+                stage: metadata.stage,
+                scope: metadata.scope ?? dataPoint.scope,
+                payloadKind: metadata.payloadKind,
+                encryptedSizeBytes: metadata.encryptedSizeBytes,
+            });
+            options.retryMemory?.recordFailure(retryKey, classified.issue.retryable);
+            if (!classified.issue.retryable) {
+                logger.warn({
+                    dataPointId: dataPoint.id,
+                    scope: classified.issue.scope,
+                    stage: classified.issue.stage,
+                    payloadKind: classified.issue.payloadKind,
+                    encryptedSizeBytes: classified.issue.encryptedSizeBytes,
+                    errorClass: classified.issue.errorClass,
+                    message: classified.issue.message,
+                }, "Quarantined corrupt synced data point");
+                continue;
+            }
             logger.error({
-                fileId: file.fileId,
-                schemaId: file.schemaId,
+                dataPointId: dataPoint.id,
+                scope: dataPoint.scope,
                 error: err.message,
-            }, "Failed to download file");
+            }, "Failed to download data point");
+            failed = true;
         }
     }
-    // 4. Advance cursor if there are new records
-    if (nextCursor !== null) {
+    // 4. Advance cursor only when every record in the page was handled.
+    if (nextCursor !== null && !failed) {
         await cursor.write(nextCursor);
     }
+    else if (failed) {
+        logger.warn({ nextCursor }, "Download cursor not advanced because one or more data points failed");
+    }
     return results;
 }
+export async function repairLocalMissingBlockSidecars(deps) {
+    const { storage, logger, diagnostics } = deps;
+    if (!storage.writeBlockManifest || !storage.hasScopeBlocks) {
+        return { repaired: 0, missingEnvelopeEntries: 0 };
+    }
+    let repaired = 0;
+    let missingEnvelopeEntries = 0;
+    let offset = 0;
+    const limit = 100;
+    while (true) {
+        const page = storage.listScopes({ limit, offset });
+        let deletedEntryInPage = false;
+        for (const scopeSummary of page.scopes) {
+            const entry = storage.findEntry({
+                scope: scopeSummary.scope,
+                at: scopeSummary.latestCollectedAt,
+            });
+            if (!entry)
+                continue;
+            const repairResult = await repairMissingBlockSidecars(storage, logger, {
+                dataPointId: entry.dataPointId ?? entry.path,
+                schemaId: entry.schemaId,
+            }, entry, diagnostics);
+            if (repairResult === "repaired")
+                repaired += 1;
+            if (repairResult === "missing-envelope") {
+                missingEnvelopeEntries += 1;
+                deletedEntryInPage = true;
+            }
+        }
+        if (!deletedEntryInPage) {
+            offset += page.scopes.length;
+        }
+        if (offset >= page.total || page.scopes.length === 0)
+            break;
+    }
+    if (repaired > 0) {
+        logger.info({ repaired }, "Repaired missing bounded block sidecars for local indexed data");
+    }
+    if (missingEnvelopeEntries > 0) {
+        logger.warn({ missingEnvelopeEntries }, "Removed stale local index entries missing local envelopes");
+    }
+    return { repaired, missingEnvelopeEntries };
+}
+function createSyncRunId() {
+    return typeof crypto !== "undefined" &&
+        typeof crypto.randomUUID === "function"
+        ? crypto.randomUUID()
+        : `download-${Date.now().toString(36)}`;
+}
+function uint8ToHex(bytes) {
+    return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+function withSyncFailureMetadata(error, metadata) {
+    if (error instanceof Error) {
+        Object.assign(error, { syncFailureMetadata: metadata });
+        return error;
+    }
+    return { error, syncFailureMetadata: metadata };
+}
+function getSyncFailureMetadata(error) {
+    if (typeof error !== "object" || error === null)
+        return {};
+    const metadata = error
+        .syncFailureMetadata;
+    if (typeof metadata !== "object" || metadata === null)
+        return {};
+    return metadata;
+}
+async function writeBlockSidecars(storage, envelope, logger, ctx, relativePath, diagnostics) {
+    if (!storage.writeBlockManifest)
+        return;
+    diagnostics?.onManifestBuildStart(ctx.dataPointId, envelope.scope);
+    try {
+        const built = await buildDataBlocksAsync({
+            scope: envelope.scope,
+            collectedAt: envelope.collectedAt,
+            schemaId: ctx.schemaId ?? undefined,
+            content: envelope,
+        });
+        await storage.writeBlockManifest(envelope.scope, envelope.collectedAt, built.manifest, built.blocks);
+        diagnostics?.onManifestBuildEnd(ctx.dataPointId, envelope.scope);
+    }
+    catch (error) {
+        const classified = classifySyncFailure({
+            error,
+            fileId: ctx.dataPointId,
+            syncRunId: "download",
+            stage: "block_build",
+            schemaId: ctx.schemaId ?? undefined,
+            scope: envelope.scope,
+        });
+        diagnostics?.onManifestBuildError(ctx.dataPointId, envelope.scope, classified.issue.message);
+        logger.warn({
+            dataPointId: ctx.dataPointId,
+            scope: envelope.scope,
+            collectedAt: envelope.collectedAt,
+            path: relativePath,
+            stage: classified.issue.stage,
+            errorClass: classified.issue.errorClass,
+            message: classified.issue.message,
+        }, "Bounded block sidecar write failed after raw envelope write");
+    }
+}
+async function repairMissingBlockSidecars(storage, logger, ctx, entry, diagnostics) {
+    if (!storage.writeBlockManifest || !storage.hasScopeBlocks) {
+        return "not-repaired";
+    }
+    let hasBlocks = false;
+    try {
+        hasBlocks = await storage.hasScopeBlocks(entry.scope, entry.collectedAt);
+    }
+    catch (error) {
+        logger.warn({
+            dataPointId: ctx.dataPointId,
+            scope: entry.scope,
+            collectedAt: entry.collectedAt,
+            error: error.message,
+        }, "Bounded block sidecar repair skipped after readiness check failed");
+        return "not-repaired";
+    }
+    if (hasBlocks)
+        return "already-ready";
+    diagnostics?.onRepair(entry.scope, "indexed entry missing block manifest; rebuilding from local envelope");
+    let envelope;
+    try {
+        envelope = await storage.readEnvelope(entry.scope, entry.collectedAt);
+    }
+    catch (error) {
+        logger.warn({
+            dataPointId: ctx.dataPointId,
+            scope: entry.scope,
+            collectedAt: entry.collectedAt,
+            error: error.message,
+        }, "Bounded block sidecar repair skipped because local envelope is unavailable");
+        // Remove the stale index entry whose payload is gone. We can only target
+        // it by gateway fileId; DataPoint-only entries (fileId === null) are left
+        // in place (no path-keyed delete on the port) and self-heal on re-download.
+        if (entry.fileId) {
+            diagnostics?.onRepair(entry.scope, "index exists without local payload; removing stale local index entry");
+            const removed = await storage.deleteByFileId(entry.fileId);
+            logger.warn({
+                dataPointId: ctx.dataPointId,
+                scope: entry.scope,
+                collectedAt: entry.collectedAt,
+                removed,
+            }, "Removed stale local index entry missing local envelope");
+            return removed ? "missing-envelope" : "not-repaired";
+        }
+        return "not-repaired";
+    }
+    await writeBlockSidecars(storage, envelope, logger, {
+        dataPointId: ctx.dataPointId,
+        schemaId: entry.schemaId ?? ctx.schemaId,
+    }, entry.path, diagnostics);
+    try {
+        return (await storage.hasScopeBlocks(entry.scope, entry.collectedAt))
+            ? "repaired"
+            : "not-repaired";
+    }
+    catch (error) {
+        logger.warn({
+            dataPointId: ctx.dataPointId,
+            scope: entry.scope,
+            collectedAt: entry.collectedAt,
+            error: error.message,
+        }, "Bounded block sidecar repair completed but readiness could not be verified");
+        return "not-repaired";
+    }
+}
 //# sourceMappingURL=download.js.map

package/dist/sync/workers/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAoBpE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAwB,EACxB,MAAkB;IAElB,MAAM,EACJ,YAAY,EACZ,gBAAgB,EAChB,cAAc,EACd,OAAO,EACP,SAAS,EACT,MAAM,GACP,GAAG,IAAI,CAAC;IAET,wDAAwD;IACxD,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,iCAAiC,CAAC,CAAC;QAC3E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4DAA4D;IAC5D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAE5D,oDAAoD;IACpD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAE1D,oDAAoD;IACpD,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAEpE,0CAA0C;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEnD,qDAAqD;IACrD,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CACrD,gBAAgB,EAChB,QAAQ,CACT,CAAC;IAEF,2CAA2C;IAC3C,YAAY,CAAC,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CACT,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,EACpE,6BAA6B,CAC9B,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAwB;IAExB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAEtD,iBAAiB;IACjB,MAAM,sBAAsB,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IAEnD,uCAAuC;IACvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,OAAO,CAAC,cAAc,CAChE,WAAW,EACX,sBAAsB,CACvB,CAAC;IAEF,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,8BAA8B;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7C,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CACV;gBACE,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAG,GAAa,CAAC,OAAO;aAC9B,EACD,yBAAyB,CAC1B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AACA,OAAO,EACL,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,GAGf,MAAM,gCAAgC,CAAC;AAIxC,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EACL,mBAAmB,EACnB,gBAAgB,GAGjB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,gBAAgB,EAA4B,MAAM,oBAAoB,CAAC;AAqFhF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAwB,EACxB,MAAuB;IAEvB,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IACzE,MAAM,GAAG,GAAkB,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAEtE,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,YAAY,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,MAAM,EACN,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EACvD,QAAQ,EACR,WAAW,CACZ,CAAC;QACF,IAAI,YAAY,KAAK,kBAAkB,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT;gBACE,WAAW,EAAE,MAAM,CAAC,EAAE;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,EACD,uEAAuE,CACxE,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,KAAK,CACV,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,EAC1B,uCAAuC,CACxC,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;IAC/D,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEjD,wEAAwE;IACxE,2EAA2E;IAC3E,oEAAoE;IACpE,2EAA2E;IAC3E,0EAA0E;IAC1E,oCAAoC;IACpC,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACxC,IAAI,SAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAI,GAAa,CAAC,OAAO,CAAC;QACtC,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,uBAAuB,CAAC,GAAG,EAAE;YACjC,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;IACL,CAAC;IACD,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAEpD,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEzC,oDAAoD;IACpD,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IACrD,IAAI,SAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,WAAW,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,kBAAkB,GAAG,SAAS,CAAC,UAAU,CAAC;QAChD,MAAM,MAAM,GAAG;YACZ,GAAa,CAAC,OAAO;YACtB,eAAe,WAAW,EAAE;YAC5B,sBAAsB,kBAAkB,EAAE;SAC3C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,uBAAuB,CAAC,GAAG,EAAE;YACjC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW;YACX,kBAAkB;SACnB,CAAC,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAyD,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5D,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAI,GAAa,CAAC,OAAO,CAAC;QACtC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC;IACzC,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAErD,MAAM,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;QAC1C,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,EAAE,EAAE,QAAQ,CAAC,WAAW;KACzB,CAAC,CAAC;IACH,IACE,iBAAiB,KAAK,SAAS;QAC/B,iBAAiB,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,EACtD,CAAC;QACD,IAAI,iBAAiB,CAAC,WAAW,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,OAAO,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,MAAM,EACN;YACE,WAAW,EAAE,MAAM,CAAC,EAAE;YACtB,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ;SACrD,EACD,iBAAiB,EACjB,WAAW,CACZ,CAAC;QACF,IAAI,YAAY,KAAK,kBAAkB,EAAE,CAAC;YACxC,MAAM,CAAC,KAAK,CACV;gBACE,WAAW,EAAE,MAAM,CAAC,EAAE;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,EACD,+CAA+C,CAChD,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,IAAI,CACT;YACE,WAAW,EAAE,MAAM,CAAC,EAAE;YACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,EACD,yEAAyE,CAC1E,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACrD,IAAI,YAAoB,CAAC;IACzB,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACH,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtE,MAAM,kBAAkB,CACtB,OAAO,EACP,QAAQ,EACR,MAAM,EACN,GAAG,EACH,YAAY,EACZ,WAAW,CACZ,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAI,GAAa,CAAC,OAAO,CAAC;QACtC,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,WAAW,CAAC;YACxB,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,SAAS;YACT,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC;YACvC,WAAW,EAAE,MAAM,CAAC,EAAE;SACvB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAI,GAAa,CAAC,OAAO,CAAC;QACtC,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEnD,MAAM,CAAC,IAAI,CACT,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,EACrE,mCAAmC,CACpC,CAAC;IAEF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAwB,EACxB,UAA8B,EAAE;IAEhC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IAEpC,MAAM,aAAa,GAAG,MAAM,+BAA+B,CAAC,IAAI,CAAC,CAAC;IAElE,kEAAkE;IAClE,MAAM,UAAU,GACd,OAAO,CAAC,aAAa,IAAI,aAAa,CAAC,sBAAsB,GAAG,CAAC;QAC/D,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IAC1B,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CACT,gFAAgF,CACjF,CAAC;IACJ,CAAC;IACD,IAAI,aAAa,CAAC,sBAAsB,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CACT,EAAE,sBAAsB,EAAE,aAAa,CAAC,sBAAsB,EAAE,EAChE,iEAAiE,CAClE,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,aAAa,IAAI,aAAa,CAAC,sBAAsB,GAAG,CAAC,EAAE,CAAC;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,oEAAoE;QACpE,uEAAuE;QACvE,mEAAmE;QACnE,sBAAsB;QACtB,OAAO,CAAC,WAAW,EAAE,cAAc,EAAE,CAAC;IACxC,CAAC;IAED,+CAA+C;IAC/C,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GACtC,MAAM,OAAO,CAAC,qBAAqB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IAE/D,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,oCAAoC;IACpC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC;QACpE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,+DAA+D;YAC/D,gEAAgE;YAChE,gEAAgE;YAChE,sCAAsC;YACtC,MAAM,CAAC,KAAK,CACV,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,EACrD,sDAAsD,CACvD,CAAC;YACF,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,oEAAoE;YACpE,wCAAwC;YACxC,MAAM,GAAG,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAClD,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC7C,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,UAAU,GAAG,mBAAmB,CAAC;gBACrC,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,SAAS,CAAC,EAAE;gBACpB,SAAS;gBACT,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK;gBACxC,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;aAChD,CAAC,CAAC;YACH,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CACT;oBACE,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK;oBAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK;oBAC7B,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,WAAW;oBACzC,kBAAkB,EAAE,UAAU,CAAC,KAAK,CAAC,kBAAkB;oBACvD,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,UAAU;oBACvC,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,OAAO;iBAClC,EACD,uCAAuC,CACxC,CAAC;gBACF,SAAS;YACX,CAAC;YACD,MAAM,CAAC,KAAK,CACV;gBACE,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAG,GAAa,CAAC,OAAO;aAC9B,EACD,+BAA+B,CAChC,CAAC;YACF,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,IAAI,UAAU,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,IAAI,CACT,EAAE,UAAU,EAAE,EACd,qEAAqE,CACtE,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,IAAoE;IAEpE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAC9C,IAAI,CAAC,OAAO,CAAC,kBAAkB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3D,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,KAAK,GAAG,GAAG,CAAC;IAElB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACnD,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC;gBAC9B,KAAK,EAAE,YAAY,CAAC,KAAK;gBACzB,EAAE,EAAE,YAAY,CAAC,iBAAiB;aACnC,CAAC,CAAC;YACH,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,MAAM,YAAY,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,MAAM,EACN;gBACE,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI;gBAC5C,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,EACD,KAAK,EACL,WAAW,CACZ,CAAC;YACF,IAAI,YAAY,KAAK,UAAU;gBAAE,QAAQ,IAAI,CAAC,CAAC;YAC/C,IAAI,YAAY,KAAK,kBAAkB,EAAE,CAAC;gBACxC,sBAAsB,IAAI,CAAC,CAAC;gBAC5B,kBAAkB,GAAG,IAAI,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC/B,CAAC;QACD,IAAI,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;IAC9D,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CACT,EAAE,QAAQ,EAAE,EACZ,gEAAgE,CACjE,CAAC;IACJ,CAAC;IACD,IAAI,sBAAsB,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CACT,EAAE,sBAAsB,EAAE,EAC1B,2DAA2D,CAC5D,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,OAAO,MAAM,KAAK,WAAW;QAClC,OAAO,MAAM,CAAC,UAAU,KAAK,UAAU;QACvC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE;QACrB,CAAC,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB;IACnC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CACzE,EAAE,CACH,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,KAAc,EACd,QAA6B;IAE7B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC;AAClD,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC3D,MAAM,QAAQ,GAAI,KAA2C;SAC1D,mBAAmB,CAAC;IACvB,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACjE,OAAO,QAA+B,CAAC;AACzC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,OAAwB,EACxB,QAKC,EACD,MAAc,EACd,GAAkB,EAClB,YAAoB,EACpB,WAAqC;IAErC,IAAI,CAAC,OAAO,CAAC,kBAAkB;QAAE,OAAO;IAExC,WAAW,EAAE,oBAAoB,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC;YACvC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,OAAO,EAAE,QAAQ;SAClB,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,kBAAkB,CAC9B,QAAQ,CAAC,KAAK,EACd,QAAQ,CAAC,WAAW,EACpB,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,MAAM,CACb,CAAC;QACF,WAAW,EAAE,kBAAkB,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,mBAAmB,CAAC;YACrC,KAAK;YACL,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,SAAS,EAAE,UAAU;YACrB,KAAK,EAAE,aAAa;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC,CAAC;QACH,WAAW,EAAE,oBAAoB,CAC/B,GAAG,CAAC,WAAW,EACf,QAAQ,CAAC,KAAK,EACd,UAAU,CAAC,KAAK,CAAC,OAAO,CACzB,CAAC;QACF,MAAM,CAAC,IAAI,CACT;YACE,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK;YAC7B,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,UAAU;YACvC,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,OAAO;SAClC,EACD,6DAA6D,CAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAwB,EACxB,MAAc,EACd,GAAkB,EAClB,KAMC,EACD,WAAqC;IAErC,IAAI,CAAC,OAAO,CAAC,kBAAkB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3D,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CACT;YACE,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,KAAK,EAAG,KAAe,CAAC,OAAO;SAChC,EACD,mEAAmE,CACpE,CAAC;QACF,OAAO,cAAc,CAAC;IACxB,CAAC;IACD,IAAI,SAAS;QAAE,OAAO,eAAe,CAAC;IAEtC,WAAW,EAAE,QAAQ,CACnB,KAAK,CAAC,KAAK,EACX,sEAAsE,CACvE,CAAC;IAEF,IAAI,QAA8D,CAAC;IACnE,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CACT;YACE,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,KAAK,EAAG,KAAe,CAAC,OAAO;SAChC,EACD,4EAA4E,CAC7E,CAAC;QACF,yEAAyE;QACzE,0EAA0E;QAC1E,4EAA4E;QAC5E,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,WAAW,EAAE,QAAQ,CACnB,KAAK,CAAC,KAAK,EACX,sEAAsE,CACvE,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3D,MAAM,CAAC,IAAI,CACT;gBACE,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,OAAO;aACR,EACD,wDAAwD,CACzD,CAAC;YACF,OAAO,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC;QACvD,CAAC;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,kBAAkB,CACtB,OAAO,EACP,QAAQ,EACR,MAAM,EACN;QACE,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ;KACzC,EACD,KAAK,CAAC,IAAI,EACV,WAAW,CACZ,CAAC;IAEF,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;YACnE,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,cAAc,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CACT;YACE,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,KAAK,EAAG,KAAe,CAAC,OAAO;SAChC,EACD,4EAA4E,CAC7E,CAAC;QACF,OAAO,cAAc,CAAC;IACxB,CAAC;AACH,CAAC"}

package/dist/sync/workers/upload.d.ts

@@ -1,13 +1,11 @@
-import type { IndexManager } from "../../storage/index/manager.js";
-import type { HierarchyManagerOptions } from "../../storage/hierarchy/index.js";
 import type { StorageAdapter } from "../../storage/adapters/interface.js";
-import type { GatewayClient } from "../../gateway/client.js";
+import { type GatewayClient } from "@opendatalabs/vana-sdk/browser";
 import type { ServerSigner } from "../../signing/signer.js";
-import type { Logger } from "pino";
+import type { Logger } from "../../logger/index.js";
 import type { IndexEntry } from "../../storage/index/types.js";
+import type { DataStoragePort } from "../../ports/index.js";
 export interface UploadWorkerDeps {
-    indexManager: IndexManager;
-    hierarchyOptions: HierarchyManagerOptions;
+    storage: DataStoragePort;
     storageAdapter: StorageAdapter;
     gateway: GatewayClient;
     signer: ServerSigner;
@@ -17,27 +15,40 @@ export interface UploadWorkerDeps {
 }
 export interface UploadResult {
     path: string;
-    fileId: string;
+    /** Storage URL the encrypted blob was written to (version-keyed). */
     url: string;
+    /**
+     * DPv2 data-point id assigned by the gateway. Present whenever this run
+     * either freshly registered the data point or found one already attached
+     * to the index entry from a prior run.
+     */
+    dataPointId: string;
+}
+export interface UploadAllOptions {
+    batchSize?: number;
+    onError?: (entry: IndexEntry, error: Error) => void;
 }
 /**
  * Upload a single unsynced index entry:
  * 1. Read local data file from disk
- * 2. Look up schema for the scope → get schemaId
- * 3. Derive scope key from master key → hex-encode as OpenPGP password
- * 4. Encrypt with OpenPGP password-based encryption → binary
- * 5. Upload OpenPGP binary to storage backend
- * 6. Sign file registration via EIP-712
- * 7. Register file record on-chain via Gateway (with schemaId)
- * 8. Update local index with fileId
+ * 2. Derive scope key from master key → hex-encode as OpenPGP password
+ * 3. Encrypt envelope with OpenPGP password-based encryption → ciphertext
+ * 4. Compute dataHash / metadataHash commitments (scope-addressed, no schema)
+ * 5. Upload ciphertext to storage backend, version-keyed `{scope}/{version}`
+ * 6. Register DPv2 data point on-chain (AddData) if not already
+ * 7. Persist the gateway-assigned dataPointId (marks the entry synced)
  */
 export declare function uploadOne(deps: UploadWorkerDeps, entry: IndexEntry): Promise<UploadResult>;
 /**
- * Process all unsynced entries (fileId === null).
+ * Process all unsynced entries (dataPointId === null).
  * Processes sequentially to avoid overwhelming storage backend.
  * Returns array of results (skips failures, logs errors).
  */
-export declare function uploadAll(deps: UploadWorkerDeps, options?: {
-    batchSize?: number;
-}): Promise<UploadResult[]>;
+export declare function uploadAll(deps: UploadWorkerDeps, options?: UploadAllOptions): Promise<UploadResult[]>;
+/**
+ * Deterministic DPv2 data-point id — `keccak256(abi.encode(owner, scope))`,
+ * the same primary key DataRegistryV2 and the gateway use. Lets the upload
+ * worker look up the registry's live row for a scope without a list call.
+ */
+export declare function computeDataPointId(ownerAddress: string, scope: string): `0x${string}`;
 //# sourceMappingURL=upload.d.ts.map

package/dist/sync/workers/upload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/upload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAChF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACnC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAK/D,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,gBAAgB,EAAE,uBAAuB,CAAC;IAC1C,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,EAAE,UAAU,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,YAAY,CAAC,CAoEvB;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,gBAAgB,EACtB,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/B,OAAO,CAAC,YAAY,EAAE,CAAC,CAkBzB"}
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/upload.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,EAAE,UAAU,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CACrD;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,YAAY,CAAC,CAwKvB;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,gBAAgB,EACtB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CAoBzB;AAeD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,GACZ,KAAK,MAAM,EAAE,CAaf"}