@indigoai-us/hq-cloud 6.18.53 → 6.18.54

package/dist/sync/state-store.js

@@ -70,6 +70,8 @@ const COMPACTION_RETRY_BACKOFF_MAX_MS = 250;
 const COMPACTION_RETRY_MIN_ATTEMPTS = 3;
 /** Safety cap so a frozen test clock cannot spin forever. */
 const COMPACTION_RETRY_MAX_ATTEMPTS = 10_000;
+/** Keep a live synchronous writer from invalidating async snapshots forever. */
+const ASYNC_COMPACTION_MAX_STALE_ATTEMPTS = 3;
 // Test instrumentation for materialisation regressions. Kept at the storage
 // seam so a fixture counts actual snapshot JSON decodes, not API invocations.
 let snapshotRecoveryCount = 0;
@@ -309,6 +311,7 @@ export class StateStore {
     walBytes;
     maxRecoveryReadBytes;
     observedHighestSnapshotGeneration;
+    deferredCompactionScheduled = false;
     constructor(options, recovered, wasCreated) {
         this.options = options;
         this.scopeDigest = digestScope(options.scopeId);
@@ -345,8 +348,16 @@ export class StateStore {
                 // A process can die after its durable append but before append() gets
                 // to its post-lock compaction. Repair that crash window before handing
                 // an already-over-budget WAL to the caller, otherwise restart loops
-                // can grow a WAL without bound.
-                store.compactIfNeeded();
+                // can grow a WAL without bound. Large domain encoders defer that
+                // repair so opening a journal on a watcher pass does not serialize its
+                // full snapshot on the runner's event loop.
+                if (normalized.snapshotEncoderAsync) {
+                    if (store.needsCompaction())
+                        store.scheduleAsyncCompaction();
+                }
+                else {
+                    store.compactIfNeeded();
+                }
                 return store;
             }
             const created = withScopeLock(dir, () => {
@@ -619,12 +630,29 @@ export class StateStore {
         return this.needsCompaction();
     }
     /** Async counterpart used by callback-originated recovery paths. */
+    scheduleAsyncCompaction() {
+        if (this.deferredCompactionScheduled)
+            return;
+        this.deferredCompactionScheduled = true;
+        setImmediate(() => {
+            void this.compactIfNeededAsync().catch((error) => {
+                process.stderr.write(`${JSON.stringify({
+                    type: "state-store-async-compaction-failed",
+                    stateDir: this.stateDir,
+                    error: error instanceof Error ? error.message : String(error),
+                })}\n`);
+            }).finally(() => {
+                this.deferredCompactionScheduled = false;
+            });
+        });
+    }
+    /** Async counterpart used by callback-originated recovery paths. */
     async compactIfNeededAsync() {
         const compacted = await withRotationClaimAsync(this.stateDir, async () => {
             const startedAt = currentTimeMs();
             const budget = this.generationPublishRetryBudgetMs();
             let attempts = 0;
-            while (attempts < COMPACTION_RETRY_MAX_ATTEMPTS) {
+            while (attempts < ASYNC_COMPACTION_MAX_STALE_ATTEMPTS) {
                 let resolveStaged;
                 let rejectStaged;
                 const stagedReady = new Promise((resolve, reject) => {
@@ -680,7 +708,7 @@ export class StateStore {
                     return false;
                 attempts += 1;
                 try {
-                    const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next, this.options.snapshotEncoder);
+                    const staged = await stageGenerationAsync(this.stateDir, this.scopeDigest, plan.next, this.options.snapshotEncoder, this.options.snapshotEncoderAsync);
                     try {
                         afterGenerationStageHookForTest?.();
                     }
@@ -705,6 +733,12 @@ export class StateStore {
             }
             return false;
         });
+        if (compacted === false && this.needsCompaction()) {
+            // A synchronous append can keep advancing the WAL while an async
+            // snapshot yields. After three rejected captures, finish one bounded
+            // rotation synchronously instead of letting the WAL grow indefinitely.
+            return this.compactIfNeeded();
+        }
         return compacted ?? false;
     }
     getGenerationIdentity() {
@@ -810,8 +844,12 @@ export class StateStore {
                 recovered = true;
                 continue;
             }
-            if (compact)
-                this.compact();
+            if (compact) {
+                if (this.options.snapshotEncoderAsync)
+                    this.scheduleAsyncCompaction();
+                else
+                    this.compact();
+            }
             return result;
         }
     }
@@ -1394,6 +1432,75 @@ function stageGeneration(dir, scopeDigest, generation, snapshotEncoder) {
         throw error;
     }
 }
+async function stageGenerationAsync(dir, scopeDigest, generation, snapshotEncoder, snapshotEncoderAsync) {
+    snapshotMaterializationCount++;
+    const payload = {
+        lastRecordSequence: generation.lastRecordSequence,
+        state: generation.state,
+    };
+    const snapshot = snapshotEncoderAsync
+        ? await snapshotEncoderAsync(generation.generation, scopeDigest, payload)
+        : encodeGenerationSnapshot(generation, scopeDigest, snapshotEncoder);
+    let snapshotTemporary;
+    let walTemporary;
+    try {
+        snapshotTemporary = await writeDurableTemporaryAsync(dir, `snapshot-${generation.generation}`, snapshot.chunks);
+        walTemporary = await writeDurableTemporaryAsync(dir, `wal-${generation.generation}`, encodeWalHeader(generation.generation, scopeDigest));
+        return { generation, snapshotTemporary, walTemporary };
+    }
+    catch (error) {
+        if (snapshotTemporary !== undefined) {
+            try {
+                await fs.promises.rm(snapshotTemporary, { force: true });
+            }
+            catch { /* preserve primary staging error */ }
+        }
+        if (walTemporary !== undefined) {
+            try {
+                await fs.promises.rm(walTemporary, { force: true });
+            }
+            catch { /* preserve primary staging error */ }
+        }
+        throw error;
+    }
+}
+async function writeDurableTemporaryAsync(dir, label, bytes) {
+    const temporary = temporaryPath(dir, label);
+    let handle;
+    try {
+        handle = await fs.promises.open(temporary, "wx");
+        if (label.startsWith("snapshot-"))
+            snapshotWriteHookForTest?.();
+        const chunks = Buffer.isBuffer(bytes) ? [bytes] : bytes;
+        for (const chunk of chunks) {
+            let offset = 0;
+            while (offset < chunk.length) {
+                const { bytesWritten } = await handle.write(chunk, offset, chunk.length - offset, null);
+                if (bytesWritten <= 0)
+                    throw new Error("state-store: async durable write made no progress");
+                offset += bytesWritten;
+            }
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+        await handle.datasync();
+        await handle.close();
+        handle = undefined;
+        return temporary;
+    }
+    catch (error) {
+        if (handle !== undefined) {
+            try {
+                await handle.close();
+            }
+            catch { /* preserve primary durable-write error */ }
+        }
+        try {
+            await fs.promises.rm(temporary, { force: true });
+        }
+        catch { /* preserve primary durable-write error */ }
+        throw new Error(`state-store: async staged write failed for ${label}`, { cause: error });
+    }
+}
 function encodeGenerationSnapshot(generation, scopeDigest, snapshotEncoder) {
     const payload = {
         lastRecordSequence: generation.lastRecordSequence,