@indigoai-us/hq-cloud 6.18.94 → 6.18.95

package/dist/sync/state-store.js

@@ -79,6 +79,15 @@ const MALFORMED_LOCK_GRACE_MS = 250;
 const SYNC_LOCK_TIMEOUT_MS = 5_000;
 const ASYNC_LOCK_TIMEOUT_MS = 60_000;
 const OPTIMISTIC_RECOVERY_MAX_RESAMPLES = 3;
+/**
+ * Off-lock recovery passes before falling back to one read under append.lock.
+ * Each failed pass means a concurrent writer moved the generation or WAL
+ * between the read and its locked validation; without a ceiling a busy writer
+ * kept a one-shot sync spinning at full CPU for hours (feedback_b7c00a62).
+ */
+const OPTIMISTIC_RECOVERY_MAX_ATTEMPTS = 8;
+/** Same ceiling for repairIfNeeded's plan loop, which then fails loudly. */
+const MAINTENANCE_PLAN_MAX_ATTEMPTS = 8;
 /** Base wall-clock budget for one contended fold. Scaled by WAL/record pressure. */
 const COMPACTION_RETRY_BASE_BUDGET_MS = 50;
 /** Append must not block indefinitely waiting to fold. */
@@ -333,6 +342,54 @@ export class StateStoreCorruptionError extends Error {
             this.rejection = options.rejection;
     }
 }
+export function readStateStoreRecoveryMarker(rootDir, scopeId) {
+    const markerPath = recoveryMarkerPath(stateDirectory(rootDir, digestScope(scopeId)));
+    try {
+        const parsed = JSON.parse(fs.readFileSync(markerPath, "utf8"));
+        if (!parsed || typeof parsed !== "object")
+            throw new Error(`state-store: invalid recovery marker at ${markerPath}`);
+        const marker = parsed;
+        if (marker.kind !== "rebuilt_from_prefix" || typeof marker.createdAt !== "string" || typeof marker.quarantinePath !== "string") {
+            throw new Error(`state-store: invalid recovery marker at ${markerPath}`);
+        }
+        return marker;
+    }
+    catch (error) {
+        if (isMissing(error))
+            return undefined;
+        throw new Error(`state-store: cannot read recovery marker at ${markerPath}`, { cause: error });
+    }
+}
+export function clearStateStoreRecoveryMarker(rootDir, scopeId) {
+    const dir = stateDirectory(rootDir, digestScope(scopeId));
+    const markerPath = recoveryMarkerPath(dir);
+    try {
+        fs.unlinkSync(markerPath);
+        fsyncDirectory(dir);
+    }
+    catch (error) {
+        if (isMissing(error))
+            return;
+        throw new Error(`state-store: cannot clear recovery marker at ${markerPath}`, { cause: error });
+    }
+}
+export class StateStoreIncompatibleGenerationError extends Error {
+    generation;
+    constructor(generation, options) {
+        super(`state-store: incompatible generation ${String(generation)}: a newer hq-cloud wrote it and this client must be upgraded before retrying. Generation files were left unchanged.`, options);
+        this.name = "StateStoreIncompatibleGenerationError";
+        this.generation = generation;
+    }
+}
+/** A reducer recognized the frame but does not recognize its WAL record type. */
+export class StateStoreUnsupportedRecordTypeError extends Error {
+    recordType;
+    constructor(recordType, reducer) {
+        super(`${reducer}: unsupported WAL record type ${String(recordType)}`);
+        this.name = "StateStoreUnsupportedRecordTypeError";
+        this.recordType = recordType;
+    }
+}
 /**
  * A snapshot whose 8-byte magic this reader does not understand.
  *
@@ -421,7 +478,15 @@ export class StateStore {
         const dir = stateDirectory(normalized.rootDir, scopeDigest);
         fs.mkdirSync(dir, { recursive: true });
         while (true) {
-            const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
+            let recovered;
+            try {
+                recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
+            }
+            catch (error) {
+                if (!normalized.recoverRejectedDelta || !isReducerRejectedGeneration(error))
+                    throw error;
+                recovered = recoverRejectedDeltaGeneration(dir, scopeDigest, normalized);
+            }
             if (recovered) {
                 const store = new StateStore(normalized, recovered, false);
                 // A process can die after its durable append but before append() gets
@@ -469,7 +534,15 @@ export class StateStore {
         const dir = stateDirectory(normalized.rootDir, scopeDigest);
         await fs.promises.mkdir(dir, { recursive: true });
         while (true) {
-            const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
+            let recovered;
+            try {
+                recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
+            }
+            catch (error) {
+                if (!normalized.recoverRejectedDelta || !isReducerRejectedGeneration(error))
+                    throw error;
+                recovered = recoverRejectedDeltaGeneration(dir, scopeDigest, normalized);
+            }
             if (recovered) {
                 const store = new StateStore(normalized, recovered, false);
                 // Match open(): a recovered over-budget WAL is compacted before this
@@ -589,7 +662,11 @@ export class StateStore {
         if (!hasSnapshotGeneration(dir))
             return { status: "missing" };
         let plan;
+        let planAttempts = 0;
         while (!plan) {
+            if (planAttempts++ >= MAINTENANCE_PLAN_MAX_ATTEMPTS) {
+                throw new StateStoreLockError("state-store maintenance did not converge: writers kept moving the store");
+            }
             const preflight = withScopeLock(dir, () => {
                 cleanupUnpublishedInitialGeneration(dir);
                 const newestGeneration = highestCompleteGenerationOnDisk(dir);
@@ -1301,6 +1378,79 @@ export class StateStore {
         this.observedHighestSnapshotGeneration = generation.observedHighestSnapshotGeneration ?? generation.generation;
     }
 }
+function isReducerRejectedGeneration(error) {
+    if (!(error instanceof StateStoreCorruptionError))
+        return false;
+    const reducerError = error.cause;
+    return reducerError instanceof StateStoreReducerError &&
+        !(reducerError.cause instanceof StateStoreUnsupportedRecordTypeError);
+}
+function recoveryMarkerPath(dir) {
+    return path.join(dir, "recovery-marker.json");
+}
+function recoverRejectedDeltaGeneration(dir, scopeDigest, options) {
+    return withScopeLock(dir, () => {
+        const generations = fs.readdirSync(dir)
+            .map((name) => /^snapshot-(\d+)\.bin$/.exec(name)?.[1])
+            .filter((value) => value !== undefined)
+            .map(Number).filter(Number.isSafeInteger).sort((a, b) => b - a);
+        const generation = highestCompleteGenerationOnDisk(dir);
+        if (generation === undefined)
+            throw new StateStoreCorruptionError("no complete generation available for rejected-delta recovery");
+        const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation, options.snapshotDecoder, options.snapshotMagics);
+        const sourceWalPath = walPath(dir, generation);
+        const parsed = readWal(sourceWalPath, scopeDigest, generation, snapshot.payload.lastRecordSequence, snapshot.payload.state, options.reduce, 0, undefined, undefined, false, true);
+        if (parsed.rejectedSuffixOffset === undefined) {
+            return {
+                generation,
+                state: parsed.state,
+                lastRecordSequence: parsed.lastRecordSequence,
+                recordCount: parsed.recordCount,
+                walBytes: parsed.walBytes,
+                maxReadBytes: parsed.maxReadBytes,
+                observedWalBytes: parsed.observedWalBytes,
+                observedHighestSnapshotGeneration: generation,
+            };
+        }
+        const rejectedSequence = parsed.lastRecordSequence + 1;
+        // The first replay may have partially mutated its state before rejection.
+        // Re-decode the snapshot so the accepted-prefix replay starts from pristine bytes.
+        const prefixSnapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation, options.snapshotDecoder, options.snapshotMagics);
+        const prefix = readWal(sourceWalPath, scopeDigest, generation, prefixSnapshot.payload.lastRecordSequence, prefixSnapshot.payload.state, (state, record) => {
+            if (record.sequence === rejectedSequence)
+                throw new Error("stop before reducer-rejected record");
+            return options.reduce(state, record);
+        }, 0, undefined, undefined, false, true);
+        if (prefix.rejectedSuffixOffset !== parsed.rejectedSuffixOffset) {
+            throw new StateStoreCorruptionError(`generation ${generation} changed during rejected-delta recovery`);
+        }
+        const suffix = fs.readFileSync(sourceWalPath).subarray(parsed.rejectedSuffixOffset);
+        const quarantinePath = path.join(dir, `wal-${generation}.rejected-${Date.now()}-${crypto.randomBytes(5).toString("hex")}.bin`);
+        try {
+            writeNewDurably(quarantinePath, suffix);
+            const marker = {
+                kind: "rebuilt_from_prefix",
+                createdAt: new Date().toISOString(),
+                quarantinePath,
+            };
+            writeDurableTempThenRename(dir, recoveryMarkerPath(dir), [Buffer.from(JSON.stringify(marker), "utf8")]);
+            const next = {
+                generation: Math.max(...generations) + 1,
+                state: prefix.state,
+                lastRecordSequence: prefix.lastRecordSequence,
+                recordCount: 0,
+                walBytes: WAL_HEADER_BYTES,
+                maxReadBytes: parsed.maxReadBytes,
+            };
+            writeGeneration(dir, scopeDigest, next, options.snapshotEncoder);
+            writeCurrentPointer(dir, next.generation);
+            return next;
+        }
+        catch (error) {
+            throw new Error(`state-store: rejected-delta recovery failed; inspect state path ${dir} and preserve its WAL files`, { cause: error });
+        }
+    });
+}
 function readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
     const names = fs.readdirSync(dir);
     const generations = names
@@ -1320,6 +1470,13 @@ function readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapsho
         }
         catch (error) {
             if (error instanceof StateStoreReducerError) {
+                if (error.cause instanceof StateStoreUnsupportedRecordTypeError) {
+                    // The frame passed its integrity checks, and this reducer explicitly
+                    // identified a record type it does not know. Do not label that as
+                    // corruption: callers may quarantine corruption errors. Never fall
+                    // back to an older generation, which would lose newer writes.
+                    throw new StateStoreIncompatibleGenerationError(generation, { cause: error });
+                }
                 // A CRC-valid record which the owning reducer cannot interpret is
                 // corrupt for that projection. Surface the public corruption contract
                 // so callers can quarantine/fail-close without depending on this
@@ -1378,7 +1535,16 @@ function readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, s
  */
 function readNewestGenerationOptimistically(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
     let resamples = 0;
+    let attempts = 0;
     while (true) {
+        if (attempts++ >= OPTIMISTIC_RECOVERY_MAX_ATTEMPTS) {
+            // Writers kept moving the store under every optimistic pass. One read
+            // under the lock always makes progress (and repairs a torn tail).
+            return withScopeLock(dir, () => {
+                cleanupUnpublishedInitialGeneration(dir);
+                return readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, true);
+            });
+        }
         const candidate = withScopeLock(dir, () => {
             cleanupUnpublishedInitialGeneration(dir);
             return highestCompleteGenerationOnDisk(dir);
@@ -1409,7 +1575,16 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, snapshotDe
 }
 async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
     let resamples = 0;
+    let attempts = 0;
     while (true) {
+        if (attempts++ >= OPTIMISTIC_RECOVERY_MAX_ATTEMPTS) {
+            // Writers kept moving the store under every optimistic pass. One read
+            // under the lock always makes progress (and repairs a torn tail).
+            return await withScopeLockAsync(dir, () => {
+                cleanupUnpublishedInitialGeneration(dir);
+                return readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, true);
+            });
+        }
         const candidate = await withScopeLockAsync(dir, () => {
             cleanupUnpublishedInitialGeneration(dir);
             return highestCompleteGenerationOnDisk(dir);
@@ -1844,7 +2019,7 @@ function readSnapshot(filePath, scopeDigest, generation, snapshotDecoder, snapsh
         fs.closeSync(fd);
     }
 }
-function readWal(filePath, scopeDigest, generation, snapshotSequence, initialState, reduce, recoveryBytesBeforeWal = 0, recoveryTotalBytes, onProgress, repairTornTail = true) {
+function readWal(filePath, scopeDigest, generation, snapshotSequence, initialState, reduce, recoveryBytesBeforeWal = 0, recoveryTotalBytes, onProgress, repairTornTail = true, stopOnReducerRejection = false) {
     let fd;
     try {
         fd = fs.openSync(filePath, "r");
@@ -1856,6 +2031,7 @@ function readWal(filePath, scopeDigest, generation, snapshotSequence, initialSta
         throw new Error(`state-store: cannot read WAL generation ${generation}`, { cause: error });
     }
     let truncateBoundary;
+    let rejectedSuffixOffset;
     try {
         const walBytes = fs.fstatSync(fd).size;
         walReadHookForTest?.();
@@ -1916,7 +2092,10 @@ function readWal(filePath, scopeDigest, generation, snapshotSequence, initialSta
                 state = reduce(state, parsed.record);
             }
             catch (error) {
-                throw new StateStoreReducerError(parsed.record, error);
+                if (!stopOnReducerRejection)
+                    throw new StateStoreReducerError(parsed.record, error);
+                rejectedSuffixOffset = offset;
+                break;
             }
             walRecordReadHookForTest?.(parsed.record);
             sequence = parsed.record.sequence;
@@ -1938,6 +2117,7 @@ function readWal(filePath, scopeDigest, generation, snapshotSequence, initialSta
             walBytes: truncateBoundary ?? walBytes,
             observedWalBytes: walBytes,
             tornWalTailBoundary: truncateBoundary,
+            rejectedSuffixOffset,
             maxReadBytes,
         };
     }