@opendatalabs/personal-server-ts-core 0.0.1-canary.01ca694 → 0.0.1-canary.052d47e

package/dist/sync/workers/download.js

@@ -1,57 +1,71 @@
 import { DataFileEnvelopeSchema, decryptWithPassword, deriveScopeKey, } from "@opendatalabs/vana-sdk/browser";
+import { feedFromGatewayClient } from "../data-point-feed.js";
+import { deletionTimestamp, isEntryCoveredByTombstone, tombstoneVersion, } from "../scope-deletions.js";
 import { buildDataBlocksAsync } from "../../storage/blocks/build.js";
 import { classifySyncFailure, inferPayloadKind, } from "../issues.js";
+import { downloadRetryKey } from "../retry-memory.js";
+import { readStoredLineage } from "../../lineage/lineage.js";
 /**
- * Download and process a single file record from the storage backend:
- * 1. Check dedup: skip if fileId already in local index
- * 2. Download OpenPGP encrypted binary from storage backend
- * 3. Resolve schemaId → scope via Gateway getSchema
+ * Download and process a single DPv2 data point from the storage backend:
+ * 1. Check dedup: skip if dataPointId already in local index
+ * 2. Reconstruct the blob URL from (scope, expectedVersion) — DataPointRecords
+ *    carry no URL, so we rebuild the version-keyed `{scope}/{version}` key the
+ *    upload worker wrote under and resolve it via storageAdapter.urlForKey
+ * 3. Download the OpenPGP encrypted binary from storage
  * 4. Derive scope key from master key → hex-encode as OpenPGP password
+ *    (scope comes straight off the record — no schema lookup needed)
  * 5. Decrypt with OpenPGP password-based decryption → plaintext JSON
  * 6. Parse as DataFileEnvelope (validate)
- * 7. Write to local filesystem via hierarchy manager
- * 8. Insert into local index (with fileId)
+ * 7. Write to local filesystem + bounded block sidecars
+ * 8. Insert into local index (with dataPointId + version)
  */
 export async function downloadOne(deps, record) {
-    const { storage, storageAdapter, gateway, masterKey, logger, diagnostics } = deps;
-    // 1. Check dedup: skip if fileId already in local index
-    const existing = storage.findByFileId(record.fileId);
+    const { storage, storageAdapter, masterKey, logger, diagnostics } = deps;
+    const ctx = { dataPointId: record.id, schemaId: null };
+    // 1. Check dedup: skip if dataPointId already in local index
+    const existing = storage.findByDataPointId(record.id);
     if (existing) {
-        await repairMissingBlockSidecars(storage, logger, record, existing, diagnostics);
-        logger.debug({ fileId: record.fileId }, "File already in index, skipping");
-        return null;
-    }
-    // 2. Download OpenPGP encrypted binary from storage backend.
-    // A download failure (404 or transient) throws and blocks the cursor — and that is correct here:
-    // the delete cascade de-registers a file BEFORE deleting its blob, so any genuinely-deleted file
-    // has deletedAt set and is excluded from this default (excludeDeleted) listing. Thus a 404 only
-    // happens for an in-flight cross-device delete, which self-heals on the next cycle once the
-    // de-register propagates and the file drops out of the list; a transient error blocks-and-retries
-    // without losing data. We deliberately do NOT skip-on-404 here: the storage layer can't reliably
-    // distinguish "gone" from "transiently unavailable", so skipping would risk a silent data gap
-    // during an outage. (Cross-device removal of already-downloaded copies is slice 3b.)
-    diagnostics?.onDownloadStart(record.fileId);
+        const repairResult = await repairMissingBlockSidecars(storage, logger, { dataPointId: record.id, schemaId: existing.schemaId }, existing, diagnostics);
+        if (repairResult === "missing-envelope") {
+            logger.warn({
+                dataPointId: record.id,
+                scope: existing.scope,
+                collectedAt: existing.collectedAt,
+            }, "Local index entry was missing its envelope; re-downloading data point");
+        }
+        else {
+            logger.debug({ dataPointId: record.id }, "Data point already in index, skipping");
+            return null;
+        }
+    }
+    // 2. Reconstruct the version-keyed storage URL.
+    const storageKey = `${record.scope}/${record.expectedVersion}`;
+    const url = storageAdapter.urlForKey(storageKey);
+    // 3. Download OpenPGP encrypted binary from storage backend. A download
+    // failure throws and blocks the cursor (correct: a transient error retries
+    // next cycle rather than risk a silent gap). Stamp the stage so the
+    // classifier can tell a deterministic 404 (quarantine, never retry) from a
+    // transient storage error — without it the failure infers stage "unknown"
+    // and every 404 is retried forever.
+    diagnostics?.onDownloadStart(record.id);
     let encrypted;
     try {
-        encrypted = await storageAdapter.download(record.url);
+        encrypted = await storageAdapter.download(url);
     }
     catch (err) {
         const detail = err.message;
-        diagnostics?.onDownloadError(record.fileId, detail);
-        throw err;
-    }
-    // 3. Resolve schemaId → scope via Gateway getSchema
-    const schema = await gateway.getSchema(record.schemaId);
-    if (!schema) {
-        diagnostics?.onDownloadError(record.fileId, `No schema found for schemaId: ${record.schemaId}`);
-        throw new Error(`No schema found for schemaId: ${record.schemaId}`);
+        diagnostics?.onDownloadError(record.id, detail);
+        throw withSyncFailureMetadata(err, {
+            stage: "download",
+            scope: record.scope,
+        });
     }
-    diagnostics?.onDownloadEnd(record.fileId, schema.scope);
-    // 4. Derive scope key → hex-encode as OpenPGP password
-    const scopeKey = deriveScopeKey(masterKey, schema.scope);
+    diagnostics?.onDownloadEnd(record.id, record.scope);
+    // 4. Derive scope key → hex-encode as OpenPGP password (scope off the record)
+    const scopeKey = deriveScopeKey(masterKey, record.scope);
     const scopeKeyHex = uint8ToHex(scopeKey);
     // 5. Decrypt with OpenPGP password-based decryption
-    diagnostics?.onDecryptStart(record.fileId, schema.scope);
+    diagnostics?.onDecryptStart(record.id, record.scope);
     let plaintext;
     try {
         plaintext = await decryptWithPassword(encrypted, scopeKeyHex);
@@ -64,9 +78,9 @@ export async function downloadOne(deps, record) {
             `payloadKind=${payloadKind}`,
             `encryptedSizeBytes=${encryptedSizeBytes}`,
         ].join("; ");
-        diagnostics?.onDecryptError(record.fileId, schema.scope, detail);
+        diagnostics?.onDecryptError(record.id, record.scope, detail);
         throw withSyncFailureMetadata(err, {
-            scope: schema.scope,
+            scope: record.scope,
             payloadKind,
             encryptedSizeBytes,
         });
@@ -79,97 +93,198 @@ export async function downloadOne(deps, record) {
     }
     catch (err) {
         const detail = err.message;
-        diagnostics?.onDecryptError(record.fileId, schema.scope, detail);
+        diagnostics?.onDecryptError(record.id, record.scope, detail);
         throw err;
     }
-    diagnostics?.onDecryptEnd(record.fileId, envelope.scope);
+    ctx.schemaId = envelope.schemaId ?? null;
+    diagnostics?.onDecryptEnd(record.id, envelope.scope);
     const existingByVersion = storage.findEntry({
         scope: envelope.scope,
         at: envelope.collectedAt,
     });
     if (existingByVersion !== undefined &&
         existingByVersion.collectedAt === envelope.collectedAt) {
-        if (existingByVersion.fileId !== record.fileId) {
-            await storage.updateFileId(existingByVersion.path, record.fileId);
+        if (existingByVersion.dataPointId !== record.id) {
+            await storage.updateDataPointId(existingByVersion.path, record.id);
         }
-        await repairMissingBlockSidecars(storage, logger, record, {
-            ...existingByVersion,
-            fileId: record.fileId,
-            schemaId: existingByVersion.schemaId ?? record.schemaId,
-        }, diagnostics);
-        logger.debug({
-            fileId: record.fileId,
+        const repairResult = await repairMissingBlockSidecars(storage, logger, {
+            dataPointId: record.id,
+            schemaId: existingByVersion.schemaId ?? ctx.schemaId,
+        }, existingByVersion, diagnostics);
+        if (repairResult !== "missing-envelope") {
+            logger.debug({
+                dataPointId: record.id,
+                scope: envelope.scope,
+                collectedAt: envelope.collectedAt,
+            }, "Data version already exists locally, skipping");
+            return null;
+        }
+        logger.warn({
+            dataPointId: record.id,
             scope: envelope.scope,
             collectedAt: envelope.collectedAt,
-        }, "File version already exists locally, skipping");
-        return null;
+        }, "Local version entry was missing its envelope; rewriting downloaded file");
     }
-    // 7. Write to local storage via the runtime storage port
-    diagnostics?.onIndexStart(record.fileId, envelope.scope);
+    // 7. Write to local storage + bounded block sidecars
+    diagnostics?.onIndexStart(record.id, envelope.scope);
     let relativePath;
     let sizeBytes;
     try {
         ({ relativePath, sizeBytes } = await storage.writeEnvelope(envelope));
-        await writeBlockSidecars(storage, envelope, logger, record, relativePath, diagnostics);
+        await writeBlockSidecars(storage, envelope, logger, ctx, relativePath, diagnostics);
     }
     catch (err) {
         const detail = err.message;
-        diagnostics?.onIndexError(record.fileId, envelope.scope, detail);
+        diagnostics?.onIndexError(record.id, envelope.scope, detail);
         throw err;
     }
-    // 8. Insert into local index (with fileId)
+    // 8. Insert into local index (with dataPointId + version)
     try {
         await storage.insertEntry({
-            fileId: record.fileId,
-            schemaId: record.schemaId,
+            fileId: null,
+            schemaId: ctx.schemaId,
             path: relativePath,
             scope: envelope.scope,
             collectedAt: envelope.collectedAt,
             sizeBytes,
+            version: Number(record.expectedVersion),
+            dataPointId: record.id,
         });
     }
     catch (err) {
         const detail = err.message;
-        diagnostics?.onIndexError(record.fileId, envelope.scope, detail);
+        diagnostics?.onIndexError(record.id, envelope.scope, detail);
         throw err;
     }
-    diagnostics?.onIndexEnd(record.fileId, envelope.scope);
-    logger.info({ fileId: record.fileId, scope: envelope.scope, path: relativePath }, "Downloaded and indexed file");
+    diagnostics?.onIndexEnd(record.id, envelope.scope);
+    logger.info({ dataPointId: record.id, scope: envelope.scope, path: relativePath }, "Downloaded and indexed data point");
+    if (deps.onDataPointIndexed) {
+        try {
+            let lineageSources;
+            try {
+                lineageSources = readStoredLineage(envelope.data)?.sources;
+            }
+            catch {
+                // A malformed $lineage is the upload worker's problem; the hook
+                // still learns about the new version.
+            }
+            deps.onDataPointIndexed({
+                scope: envelope.scope,
+                dataPointId: record.id,
+                version: Number(record.expectedVersion),
+                collectedAt: envelope.collectedAt,
+                lineageSources,
+            });
+        }
+        catch (err) {
+            logger.warn({ scope: envelope.scope, error: err.message }, "onDataPointIndexed hook failed; data point already indexed");
+        }
+    }
     return {
-        fileId: record.fileId,
+        dataPointId: record.id,
         scope: envelope.scope,
         collectedAt: envelope.collectedAt,
         path: relativePath,
     };
 }
 /**
- * Poll Gateway for new file records since lastProcessedTimestamp,
- * download each, and advance the cursor only when the page fully succeeds.
+ * Poll Gateway for the owner's data points since the stored cursor, download
+ * each, and advance the cursor only when the page fully succeeds.
  */
-export async function downloadAll(deps) {
+export async function downloadAll(deps, options = {}) {
     const { gateway, cursor, serverOwner, logger } = deps;
     const syncRunId = createSyncRunId();
-    await repairLocalMissingBlockSidecars(deps);
-    // 1. Read cursor
-    const lastProcessedTimestamp = await cursor.read();
-    // 2. Poll gateway for new file records, including soft-deleted ones so we can reconcile deletions
-    // of files this device already holds (cross-device "deleted on desktop → gone on web too").
-    const { files, cursor: nextCursor } = await gateway.listFilesSince(serverOwner, lastProcessedTimestamp, { includeDeleted: true });
+    const repairSummary = await repairLocalMissingBlockSidecars(deps);
+    // 1. Read cursor (opaque pagination cursor persisted across runs)
+    const lastCursor = options.fullReconcile || repairSummary.missingEnvelopeEntries > 0
+        ? null
+        : await cursor.read();
+    if (options.fullReconcile) {
+        logger.info("Running full registry reconciliation to repair any missing local scope records");
+    }
+    if (repairSummary.missingEnvelopeEntries > 0) {
+        logger.warn({ missingEnvelopeEntries: repairSummary.missingEnvelopeEntries }, "Resetting this sync listing to repair stale local index entries");
+    }
+    if (options.fullReconcile || repairSummary.missingEnvelopeEntries > 0) {
+        // An EXPLICIT listing restart (full reconcile, or the repair path above)
+        // exists to re-fetch — give transient failures a fresh attempt budget so
+        // it can; 404s stay dead. Note `lastCursor === null` alone is NOT a
+        // reset signal: single-page listings never advance the cursor, so null
+        // is their steady state and clearing on it would erase the backoff
+        // memory every cycle.
+        options.retryMemory?.onListingReset();
+    }
+    // 2. Poll gateway for the owner's data points, tombstones included: a
+    // deletion made on another replica must reach this one or its local copy
+    // would be re-uploaded and resurrect the point.
+    const feed = deps.dataPointFeed ?? feedFromGatewayClient(gateway);
+    const { dataPoints, cursor: nextCursor } = await feed.listDataPointsByOwner(serverOwner, lastCursor, { includeDeleted: true });
+    // Feed the read-side deletion memory before any per-record work so a
+    // tombstone is refused by reads even if its local reconcile below fails.
+    // Only a listing with no further pages proves the memory complete: a
+    // multi-page backlog leaves it "stale" and reads fall back to lookups. A
+    // listing from no cursor covered the whole registry (`full`), which is
+    // what re-validates remembered tombstones; an incremental pass does not.
+    if (deps.scopeDeletions) {
+        for (const dataPoint of dataPoints) {
+            const deletedAt = deletionTimestamp(dataPoint);
+            if (deletedAt !== null) {
+                deps.scopeDeletions.markDeleted(dataPoint.scope, {
+                    deletedAt,
+                    version: tombstoneVersion(dataPoint),
+                });
+            }
+            else {
+                deps.scopeDeletions.markLive(dataPoint.scope);
+            }
+        }
+        if (nextCursor === null) {
+            deps.scopeDeletions.noteFeedSynced(undefined, {
+                full: lastCursor === null,
+            });
+        }
+    }
     const results = [];
     let failed = false;
-    // 3. Process each file record
-    for (const file of files) {
-        try {
-            // Deletion reconciliation: a record marked deletedAt means the owner deleted it elsewhere.
-            // Drop any local copy instead of downloading. Idempotent — no local copy is a no-op.
-            if (file.deletedAt) {
-                const removed = await deps.storage.deleteByFileId(file.fileId);
-                if (removed) {
-                    logger.info({ fileId: file.fileId }, "Reconciled remote deletion: removed local copy");
-                }
-                continue;
+    // 3. Process each data point record
+    for (const dataPoint of dataPoints) {
+        const deletedAt = deletionTimestamp(dataPoint);
+        if (deletedAt !== null) {
+            // Tombstone: never download; drop what the deletion covers locally.
+            // A failure here blocks the cursor so the tombstone is re-listed and
+            // the reconcile retried next cycle.
+            try {
+                await reconcileDeletedDataPoint(deps, dataPoint, deletedAt);
+            }
+            catch (err) {
+                logger.error({
+                    dataPointId: dataPoint.id,
+                    scope: dataPoint.scope,
+                    error: err.message,
+                }, "Failed to reconcile deleted data point locally");
+                failed = true;
             }
-            const result = await downloadOne(deps, file);
+            continue;
+        }
+        const retryKey = downloadRetryKey(dataPoint);
+        const decision = options.retryMemory?.decide(retryKey) ?? "attempt";
+        if (decision === "give-up") {
+            // Non-retryable failure (e.g. blob 404) or exhausted transient
+            // retries — skip without blocking the cursor so one dead record
+            // can't wedge the rest of the listing. A new registered version
+            // (different retry key) starts fresh.
+            logger.debug({ dataPointId: dataPoint.id, scope: dataPoint.scope }, "Skipping data point after exhausted download retries");
+            continue;
+        }
+        if (decision === "backoff") {
+            // Waiting out the backoff window. Block the cursor so the record is
+            // still listed when the window expires.
+            failed = true;
+            continue;
+        }
+        try {
+            const result = await downloadOne(deps, dataPoint);
+            options.retryMemory?.recordSuccess(retryKey);
             if (result) {
                 results.push(result);
             }
@@ -178,32 +293,31 @@ export async function downloadAll(deps) {
             const metadata = getSyncFailureMetadata(err);
             const classified = classifySyncFailure({
                 error: err,
-                fileId: file.fileId,
-                schemaId: file.schemaId,
+                fileId: dataPoint.id,
                 syncRunId,
                 stage: metadata.stage,
-                scope: metadata.scope,
+                scope: metadata.scope ?? dataPoint.scope,
                 payloadKind: metadata.payloadKind,
                 encryptedSizeBytes: metadata.encryptedSizeBytes,
             });
+            options.retryMemory?.recordFailure(retryKey, classified.issue.retryable);
             if (!classified.issue.retryable) {
                 logger.warn({
-                    fileId: file.fileId,
-                    schemaId: file.schemaId,
+                    dataPointId: dataPoint.id,
                     scope: classified.issue.scope,
                     stage: classified.issue.stage,
                     payloadKind: classified.issue.payloadKind,
                     encryptedSizeBytes: classified.issue.encryptedSizeBytes,
                     errorClass: classified.issue.errorClass,
                     message: classified.issue.message,
-                }, "Quarantined corrupt synced file");
+                }, "Quarantined corrupt synced data point");
                 continue;
             }
             logger.error({
-                fileId: file.fileId,
-                schemaId: file.schemaId,
+                dataPointId: dataPoint.id,
+                scope: dataPoint.scope,
                 error: err.message,
-            }, "Failed to download file");
+            }, "Failed to download data point");
             failed = true;
         }
     }
@@ -212,19 +326,78 @@ export async function downloadAll(deps) {
         await cursor.write(nextCursor);
     }
     else if (failed) {
-        logger.warn({ nextCursor }, "Download cursor not advanced because one or more files failed");
+        logger.warn({ nextCursor }, "Download cursor not advanced because one or more data points failed");
     }
     return results;
 }
+/**
+ * Apply a gateway tombstone to the local index: remove every local version
+ * the tombstone covers (synced rows at or below its version, and unsynced
+ * rows ingested without knowledge of it), keep unsynced rows ingested on top
+ * of it (a legitimate re-add that the upload worker registers after the
+ * tombstone version). See `isEntryCoveredByTombstone`.
+ */
+export async function reconcileDeletedDataPoint(deps, record, deletedAt) {
+    const tombstone = { version: tombstoneVersion(record) };
+    const tombstoned = tombstone.version === null ? null : BigInt(tombstone.version);
+    // Unsynced covered entries above the tombstone version: the deleting
+    // replica's exact-key pass only enumerates registry versions, so any
+    // ciphertext this replica uploaded under a higher, never-registered key is
+    // ours to clean up.
+    const orphanKeys = [];
+    const { storage, logger } = deps;
+    const PAGE_SIZE = 500;
+    const stale = [];
+    let kept = 0;
+    for (let offset = 0;; offset += PAGE_SIZE) {
+        const entries = storage.listVersions(record.scope, {
+            limit: PAGE_SIZE,
+            offset,
+        });
+        for (const entry of entries) {
+            if (isEntryCoveredByTombstone(entry, tombstone)) {
+                stale.push({ scope: entry.scope, collectedAt: entry.collectedAt });
+                if (entry.dataPointId === null &&
+                    (tombstoned === null || BigInt(entry.version) > tombstoned)) {
+                    orphanKeys.push({
+                        scope: entry.scope,
+                        version: String(entry.version),
+                    });
+                }
+            }
+            else {
+                kept += 1;
+            }
+        }
+        if (entries.length < PAGE_SIZE)
+            break;
+    }
+    let removed = 0;
+    for (const version of stale) {
+        if (await storage.deleteVersion(version.scope, version.collectedAt)) {
+            removed += 1;
+        }
+    }
+    if (orphanKeys.length > 0 && deps.pendingBlobDeletions) {
+        await deps.pendingBlobDeletions.add(orphanKeys);
+    }
+    if (removed > 0 || kept > 0) {
+        logger.info({ dataPointId: record.id, scope: record.scope, deletedAt, removed, kept }, "Reconciled gateway deletion against local index");
+    }
+    return { scope: record.scope, deletedAt, removed, kept };
+}
 export async function repairLocalMissingBlockSidecars(deps) {
     const { storage, logger, diagnostics } = deps;
-    if (!storage.writeBlockManifest || !storage.hasScopeBlocks)
-        return 0;
+    if (!storage.writeBlockManifest || !storage.hasScopeBlocks) {
+        return { repaired: 0, missingEnvelopeEntries: 0 };
+    }
     let repaired = 0;
+    let missingEnvelopeEntries = 0;
     let offset = 0;
     const limit = 100;
     while (true) {
         const page = storage.listScopes({ limit, offset });
+        let deletedEntryInPage = false;
         for (const scopeSummary of page.scopes) {
             const entry = storage.findEntry({
                 scope: scopeSummary.scope,
@@ -232,24 +405,30 @@ export async function repairLocalMissingBlockSidecars(deps) {
             });
             if (!entry)
                 continue;
-            const repairedScope = await repairMissingBlockSidecars(storage, logger, {
-                fileId: entry.fileId ?? entry.path,
-                owner: "",
-                url: "",
-                schemaId: entry.schemaId ?? "",
-                createdAt: entry.createdAt,
+            const repairResult = await repairMissingBlockSidecars(storage, logger, {
+                dataPointId: entry.dataPointId ?? entry.path,
+                schemaId: entry.schemaId,
             }, entry, diagnostics);
-            if (repairedScope)
+            if (repairResult === "repaired")
                 repaired += 1;
+            if (repairResult === "missing-envelope") {
+                missingEnvelopeEntries += 1;
+                deletedEntryInPage = true;
+            }
+        }
+        if (!deletedEntryInPage) {
+            offset += page.scopes.length;
         }
-        offset += page.scopes.length;
         if (offset >= page.total || page.scopes.length === 0)
             break;
     }
     if (repaired > 0) {
         logger.info({ repaired }, "Repaired missing bounded block sidecars for local indexed data");
     }
-    return repaired;
+    if (missingEnvelopeEntries > 0) {
+        logger.warn({ missingEnvelopeEntries }, "Removed stale local index entries missing local envelopes");
+    }
+    return { repaired, missingEnvelopeEntries };
 }
 function createSyncRunId() {
     return typeof crypto !== "undefined" &&
@@ -276,32 +455,32 @@ function getSyncFailureMetadata(error) {
         return {};
     return metadata;
 }
-async function writeBlockSidecars(storage, envelope, logger, record, relativePath, diagnostics) {
+async function writeBlockSidecars(storage, envelope, logger, ctx, relativePath, diagnostics) {
     if (!storage.writeBlockManifest)
         return;
-    diagnostics?.onManifestBuildStart(record.fileId, envelope.scope);
+    diagnostics?.onManifestBuildStart(ctx.dataPointId, envelope.scope);
     try {
         const built = await buildDataBlocksAsync({
             scope: envelope.scope,
             collectedAt: envelope.collectedAt,
-            schemaId: record.schemaId,
+            schemaId: ctx.schemaId ?? undefined,
             content: envelope,
         });
         await storage.writeBlockManifest(envelope.scope, envelope.collectedAt, built.manifest, built.blocks);
-        diagnostics?.onManifestBuildEnd(record.fileId, envelope.scope);
+        diagnostics?.onManifestBuildEnd(ctx.dataPointId, envelope.scope);
     }
     catch (error) {
         const classified = classifySyncFailure({
             error,
-            fileId: record.fileId,
+            fileId: ctx.dataPointId,
             syncRunId: "download",
             stage: "block_build",
-            schemaId: record.schemaId,
+            schemaId: ctx.schemaId ?? undefined,
             scope: envelope.scope,
         });
-        diagnostics?.onManifestBuildError(record.fileId, envelope.scope, classified.issue.message);
+        diagnostics?.onManifestBuildError(ctx.dataPointId, envelope.scope, classified.issue.message);
         logger.warn({
-            fileId: record.fileId,
+            dataPointId: ctx.dataPointId,
             scope: envelope.scope,
             collectedAt: envelope.collectedAt,
             path: relativePath,
@@ -311,24 +490,25 @@ async function writeBlockSidecars(storage, envelope, logger, record, relativePat
         }, "Bounded block sidecar write failed after raw envelope write");
     }
 }
-async function repairMissingBlockSidecars(storage, logger, record, entry, diagnostics) {
-    if (!storage.writeBlockManifest || !storage.hasScopeBlocks)
-        return false;
+async function repairMissingBlockSidecars(storage, logger, ctx, entry, diagnostics) {
+    if (!storage.writeBlockManifest || !storage.hasScopeBlocks) {
+        return "not-repaired";
+    }
     let hasBlocks = false;
     try {
         hasBlocks = await storage.hasScopeBlocks(entry.scope, entry.collectedAt);
     }
     catch (error) {
         logger.warn({
-            fileId: entry.fileId ?? record.fileId,
+            dataPointId: ctx.dataPointId,
             scope: entry.scope,
             collectedAt: entry.collectedAt,
             error: error.message,
         }, "Bounded block sidecar repair skipped after readiness check failed");
-        return false;
+        return "not-repaired";
     }
     if (hasBlocks)
-        return false;
+        return "already-ready";
     diagnostics?.onRepair(entry.scope, "indexed entry missing block manifest; rebuilding from local envelope");
     let envelope;
     try {
@@ -336,29 +516,44 @@ async function repairMissingBlockSidecars(storage, logger, record, entry, diagno
     }
     catch (error) {
         logger.warn({
-            fileId: entry.fileId ?? record.fileId,
+            dataPointId: ctx.dataPointId,
             scope: entry.scope,
             collectedAt: entry.collectedAt,
             error: error.message,
         }, "Bounded block sidecar repair skipped because local envelope is unavailable");
-        return false;
+        // Remove the stale index entry whose payload is gone. We can only target
+        // it by gateway fileId; DataPoint-only entries (fileId === null) are left
+        // in place (no path-keyed delete on the port) and self-heal on re-download.
+        if (entry.fileId) {
+            diagnostics?.onRepair(entry.scope, "index exists without local payload; removing stale local index entry");
+            const removed = await storage.deleteByFileId(entry.fileId);
+            logger.warn({
+                dataPointId: ctx.dataPointId,
+                scope: entry.scope,
+                collectedAt: entry.collectedAt,
+                removed,
+            }, "Removed stale local index entry missing local envelope");
+            return removed ? "missing-envelope" : "not-repaired";
+        }
+        return "not-repaired";
     }
     await writeBlockSidecars(storage, envelope, logger, {
-        ...record,
-        fileId: entry.fileId ?? record.fileId,
-        schemaId: entry.schemaId ?? record.schemaId,
+        dataPointId: ctx.dataPointId,
+        schemaId: entry.schemaId ?? ctx.schemaId,
     }, entry.path, diagnostics);
     try {
-        return await storage.hasScopeBlocks(entry.scope, entry.collectedAt);
+        return (await storage.hasScopeBlocks(entry.scope, entry.collectedAt))
+            ? "repaired"
+            : "not-repaired";
     }
     catch (error) {
         logger.warn({
-            fileId: entry.fileId ?? record.fileId,
+            dataPointId: ctx.dataPointId,
             scope: entry.scope,
             collectedAt: entry.collectedAt,
             error: error.message,
         }, "Bounded block sidecar repair completed but readiness could not be verified");
-        return false;
+        return "not-repaired";
     }
 }
 //# sourceMappingURL=download.js.map