@indigoai-us/hq-cloud 6.18.52 → 6.18.54

package/dist/journal.js

@@ -20,11 +20,12 @@ import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
 import { createJournalRecordProxy, journalRecordRows } from "./journal-record-proxy.js";
+import { fsyncWithCompatibility } from "./sync/fsync-compat.js";
 import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
 import { forgetSharedAreaJournalRows, } from "./journal-snapshot-rows.js";
 export { JOURNAL_CACHE_MAX_ROWS, JOURNAL_DECODE_WINDOW, dropJournalDecodeWindow, emitJournalCacheMetric, journalCacheMetric, journalFingerprintMatchesStat, resetJournalDecodeWindowForTest, } from "./journal-decode-window.js";
 import { journalCacheMetric } from "./journal-decode-window.js";
-import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
+import { encodeV4JournalSnapshotChunks, encodeV4JournalSnapshotChunksAsync, } from "./journal-snapshot-v4.js";
 import { toPosixKey } from "./posix-key.js";
 import { isUnderAnyRoot } from "./path-ancestors.js";
 export { isUnderAnyRoot } from "./path-ancestors.js";
@@ -60,6 +61,16 @@ const journalBaselines = new WeakMap();
 const storeBackedJournals = new WeakMap();
 /** Full-pass journals must not retain the post-cutover v3 rollback mirror between checkpoints. */
 const fullPassAreaJournals = new WeakSet();
+// Large asynchronous snapshot capture is enabled only by the long-lived watch
+// runner. One-shot journal writers keep the synchronous writer's lower
+// transient-memory profile.
+let asyncJournalCompactionEnabled = false;
+export function setAsyncJournalCompactionForWatchRunner(watchMode) {
+    asyncJournalCompactionEnabled = watchMode;
+}
+export function enableAsyncJournalCompactionForWatchRunner() {
+    setAsyncJournalCompactionForWatchRunner(true);
+}
 /**
  * Record-shaped journals are a public compatibility boundary. Internal callers
  * use this adapter so packed rows stay lazy through reads, scans and writes.
@@ -506,6 +517,42 @@ function sanitizeSlug(slug) {
 export function getJournalPath(slug) {
     return path.join(getStateDir(), `${JOURNAL_FILE_PREFIX}${sanitizeSlug(slug)}${JOURNAL_FILE_SUFFIX}`);
 }
+/**
+ * Paths retained in the durable quarantine sidecars for this journal scope.
+ * Ordinary quarantined delta paths remain blocked until repaired. An area row
+ * marked `freshCompare` has no trustworthy baseline, so sync may compare it as
+ * an unsynced path while still retaining its quarantine evidence.
+ */
+export function readQuarantinedJournalPaths(slug) {
+    const scope = crypto.createHash("sha256").update(journalScopeId(slug)).digest("hex");
+    const directory = path.join(getStateDir(), "journal-quarantine", scope);
+    let names;
+    try {
+        names = fs.readdirSync(directory);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return new Set();
+        throw error;
+    }
+    const paths = new Set();
+    for (const name of names) {
+        if (!name.endsWith(".json"))
+            continue;
+        const sidecar = JSON.parse(fs.readFileSync(path.join(directory, name), "utf8"));
+        if (sidecar.format !== "hq-journal-quarantine@1" || !Array.isArray(sidecar.invalidUpserts)) {
+            throw new Error("journal: invalid quarantine sidecar");
+        }
+        if (sidecar.freshCompare === true)
+            continue;
+        for (const invalid of sidecar.invalidUpserts) {
+            if (typeof invalid.key !== "string")
+                throw new Error("journal: invalid quarantined path");
+            paths.add(invalid.key);
+        }
+    }
+    return paths;
+}
 /**
  * Reserved journal slug for the personal-vault fanout slot in the `--companies`
  * runner. The vault slot uploads the whole HQ overlay (`.claude/`, `core/`,
@@ -571,7 +618,7 @@ export function migratePersonalVaultJournal() {
  * When the file doesn't exist, we return a fresh v2 journal directly — new
  * installs never pass through v1 on disk.
  */
-export function readJournal(slug) {
+export function readJournal(slug, companyUid) {
     const areaMigration = openAreaMigrationIfAuthoritative(slug);
     if (areaMigration) {
         // Area rows stay in their shared packed store. This record-compatible
@@ -583,7 +630,7 @@ export function readJournal(slug) {
     // same immutable packed rows. Each owns only its own overlay, so an
     // uncommitted full pass is never visible to a scoped reader (and a scoped
     // delta cannot dirty the full pass).
-    return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
+    return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true, companyUid), { privateOverlay: true });
 }
 /**
  * Read the mutable outer shell required by a full pull without detaching every
@@ -599,10 +646,10 @@ export function readJournal(slug) {
  *
  * @internal Full-pull boundary; not a public journal record API.
  */
-export function readJournalForPass(slug) {
+export function readJournalForPass(slug, companyUid) {
     const areaMigration = openAreaMigrationIfAuthoritative(slug);
     if (!areaMigration)
-        return readJournal(slug);
+        return readJournal(slug, companyUid);
     // The overlay owns only keys this pass changes; it borrows all untouched
     // packed rows from the cached aggregate and keeps the public record facade.
     const journal = readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
@@ -631,12 +678,12 @@ export function readJournalForPass(slug) {
 function readJournalAggregate(slug) {
     return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
 }
-function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false) {
+function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false, companyUid) {
     if (areaMigration)
         return areaMigration.readAreaJournal();
     const store = StateStore.exists(getStateDir(), journalScopeId(slug))
-        ? openJournalStore(slug, emptyJournal())
-        : openJournalStore(slug, readLegacyJournal(slug));
+        ? openJournalStore(slug, emptyJournal(), companyUid)
+        : openJournalStore(slug, readLegacyJournal(slug), companyUid);
     if (retainScopedProjection) {
         // A full pass has already paid to recover this packed snapshot. Retain its
         // projection for the next scoped pass instead of making that pass decode
@@ -805,7 +852,7 @@ export function setJournalSnapshotDecoderForTest(decoder) {
 export function setJournalSnapshotEncoderForTest(encoder) {
     journalSnapshotEncoderForTest = encoder;
 }
-function journalStoreOptions(slug, initialJournal) {
+function journalStoreOptions(slug, initialJournal, companyUid) {
     return {
         rootDir: getStateDir(),
         scopeId: journalScopeId(slug),
@@ -816,6 +863,9 @@ function journalStoreOptions(slug, initialJournal) {
         snapshotEncoder: journalSnapshotEncoderForTest === undefined
             ? (generation, scopeDigest, payload) => encodeV4JournalSnapshotChunks(generation, scopeDigest, payload, (diagnostic) => journalSnapshotQuarantineListeners.get(slug)?.(diagnostic))
             : journalSnapshotEncoderForTest ?? undefined,
+        snapshotEncoderAsync: asyncJournalCompactionEnabled && journalSnapshotEncoderForTest === undefined
+            ? (generation, scopeDigest, payload) => encodeV4JournalSnapshotChunksAsync(generation, scopeDigest, payload, (diagnostic) => journalSnapshotQuarantineListeners.get(slug)?.(diagnostic))
+            : undefined,
         snapshotMagics: [SNAPSHOT_V4_MAGIC],
         validateRecord: validateJournalRecord,
         reduce: (state, record) => {
@@ -828,6 +878,7 @@ function journalStoreOptions(slug, initialJournal) {
             }
             if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
                 const delta = record.payload;
+                const replayDelta = quarantineIncompleteJournalUpserts(slug, companyUid, record, delta);
                 const existingRows = journalRecordRows(state.journal.files) ?? decodedJournalRows(state.journal);
                 if (existingRows) {
                     // StateStore's aggregate is also the immutable source for any
@@ -835,14 +886,14 @@ function journalStoreOptions(slug, initialJournal) {
                     // observed, put durable changes in a separate overlay rather than
                     // mutating that shared packed row store.
                     const rows = existingRows instanceof JournalRowOverlay
-                        ? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
-                        : new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
+                        ? existingRows.withCommittedDelta(replayDelta.upserts, replayDelta.deletes)
+                        : new JournalRowOverlay(existingRows).withCommittedDelta(replayDelta.upserts, replayDelta.deletes);
                     state.journal.files = createJournalRecordProxy(rows);
                 }
                 else {
-                    for (const key of delta.deletes)
+                    for (const key of replayDelta.deletes)
                         delete state.journal.files[key];
-                    Object.assign(state.journal.files, delta.upserts);
+                    Object.assign(state.journal.files, replayDelta.upserts);
                 }
                 state.journal.version = delta.version;
                 state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
@@ -861,12 +912,14 @@ function validateJournalRecord(record) {
         }
         return;
     }
-    if (record.type === JOURNAL_DELTA_RECORD_TYPE && isJournalDelta(record.payload))
+    if (record.type === JOURNAL_DELTA_RECORD_TYPE && isJournalDelta(record.payload)) {
+        assertCompleteJournalUpserts(record.payload);
         return;
+    }
     throw new Error(`journal: unexpected v3 WAL record type ${record.type}`);
 }
-function openJournalStore(slug, initialJournal) {
-    return StateStore.open(journalStoreOptions(slug, initialJournal));
+function openJournalStore(slug, initialJournal, companyUid) {
+    return StateStore.open(journalStoreOptions(slug, initialJournal, companyUid));
 }
 /**
  * A committed area manifest is an authority decision, not a hint.  If its
@@ -913,6 +966,7 @@ function openAreaMigrationIfAuthoritative(slug) {
         rootDir: stateDir,
         journalSlug: slug,
         resolver,
+        onIncompleteJournalRow: (key, entry) => quarantineIncompleteAreaJournalRow(slug, key, entry),
         // Area authority is already committed. Open the full reverse-shadow v3
         // projection only during a mirror mutation instead of pinning it next to
         // the authoritative area ledger for the runner's whole lifetime.
@@ -1245,6 +1299,7 @@ export class JournalStore {
             upserts: structuredClone(delta.upserts ?? {}),
             deletes: [...(delta.deletes ?? [])],
         };
+        assertCompleteJournalUpserts(payload);
         if (delta.metadata?.lastSync !== undefined)
             payload.lastSync = delta.metadata.lastSync;
         if (delta.metadata?.pulls !== undefined)
@@ -1275,6 +1330,7 @@ export class JournalStore {
                 payload.lastSync = delta.metadata.lastSync;
             if (delta.metadata?.pulls !== undefined)
                 payload.pulls = structuredClone(delta.metadata.pulls);
+            assertCompleteJournalUpserts(payload);
             return payload;
         };
         if (this.areaMigration) {
@@ -1312,6 +1368,7 @@ export class JournalStore {
             upserts: structuredClone(delta.upserts ?? {}),
             deletes: [...(delta.deletes ?? [])],
         };
+        assertCompleteJournalUpserts(payload);
         if (delta.metadata?.lastSync !== undefined)
             payload.lastSync = delta.metadata.lastSync;
         if (delta.metadata?.pulls !== undefined)
@@ -1591,6 +1648,180 @@ function isJournalDelta(value) {
         Array.isArray(delta.deletes) && delta.deletes.every((key) => typeof key === "string") &&
         (delta.pulls === undefined || Array.isArray(delta.pulls));
 }
+export class JournalIncompleteUpsertError extends Error {
+    missingFields;
+    errorClass = "JournalIncompleteUpsertError";
+    constructor(missingFields) {
+        super(`journal delta upsert is incomplete (${missingFields.join(",")})`);
+        this.missingFields = missingFields;
+        this.name = "JournalIncompleteUpsertError";
+    }
+}
+function incompleteJournalEntryFields(value) {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        return ["hash", "size", "syncedAt", "direction"];
+    }
+    const entry = value;
+    const missing = [];
+    if (typeof entry.hash !== "string")
+        missing.push("hash");
+    if (typeof entry.size !== "number" || !Number.isFinite(entry.size))
+        missing.push("size");
+    if (typeof entry.syncedAt !== "string")
+        missing.push("syncedAt");
+    if (entry.direction !== "up" && entry.direction !== "down")
+        missing.push("direction");
+    return missing;
+}
+function assertCompleteJournalUpserts(delta) {
+    for (const entry of Object.values(delta.upserts)) {
+        const missing = incompleteJournalEntryFields(entry);
+        if (missing.length > 0)
+            throw new JournalIncompleteUpsertError(missing);
+    }
+}
+const loggedQuarantinedJournalRecords = new Set();
+function quarantineIncompleteJournalUpserts(slug, companyUid, record, delta, freshCompare = false) {
+    const invalid = Object.entries(delta.upserts)
+        .map(([key, value]) => ({ key, value, missingFields: incompleteJournalEntryFields(value) }))
+        .filter((entry) => entry.missingFields.length > 0);
+    if (invalid.length === 0)
+        return delta;
+    const content = JSON.stringify({
+        format: "hq-journal-quarantine@1",
+        slug,
+        companyUid,
+        sequence: record.sequence,
+        recordType: record.type,
+        ...(freshCompare ? { freshCompare: true } : {}),
+        invalidUpserts: invalid,
+    });
+    const digest = crypto.createHash("sha256").update(content).digest("hex");
+    const quarantineDirectory = path.join(getStateDir(), "journal-quarantine", crypto.createHash("sha256").update(journalScopeId(slug)).digest("hex"));
+    const finalPath = path.join(quarantineDirectory, `${record.sequence}-${digest}.json`);
+    let linkAttempted = false;
+    try {
+        fs.mkdirSync(quarantineDirectory, { recursive: true, mode: 0o700 });
+        const tempPath = path.join(quarantineDirectory, `${record.sequence}-${digest}-${crypto.randomBytes(8).toString("hex")}.json.tmp`);
+        let descriptor;
+        let operationError;
+        let operationFailed = false;
+        let cleanupError;
+        let cleanupFailed = false;
+        try {
+            descriptor = fs.openSync(tempPath, "wx", 0o600);
+            fs.writeFileSync(descriptor, `${content}\n`);
+            fs.fsyncSync(descriptor);
+            fs.closeSync(descriptor);
+            descriptor = undefined;
+            // A hard link publishes the fully synced inode atomically and fails if a
+            // previous replay already installed this immutable record. Unlike rename,
+            // link never overwrites an existing complete quarantine sidecar.
+            linkAttempted = true;
+            fs.linkSync(tempPath, finalPath);
+        }
+        catch (error) {
+            operationError = error;
+            operationFailed = true;
+        }
+        finally {
+            if (descriptor !== undefined) {
+                try {
+                    fs.closeSync(descriptor);
+                }
+                catch (error) {
+                    cleanupError = error;
+                    cleanupFailed = true;
+                }
+            }
+            try {
+                fs.unlinkSync(tempPath);
+            }
+            catch (error) {
+                if (error.code !== "ENOENT" && !cleanupFailed) {
+                    cleanupError = error;
+                    cleanupFailed = true;
+                }
+            }
+        }
+        // Keep the operation's failure authoritative if cleanup also fails. The
+        // cleanup error is surfaced only when publishing itself succeeded.
+        if (operationFailed)
+            throw operationError;
+        if (cleanupFailed)
+            throw cleanupError;
+        const directoryDescriptor = fs.openSync(quarantineDirectory, "r");
+        try {
+            fsyncWithCompatibility(directoryDescriptor, {
+                site: "journal quarantine directory",
+                handleKind: "directory",
+            });
+        }
+        finally {
+            fs.closeSync(directoryDescriptor);
+        }
+    }
+    catch (error) {
+        // Replay may proceed without a write only when the same immutable sidecar
+        // was already published. Other persistence failures must stop replay so a
+        // later compaction cannot discard the only copy of this malformed record.
+        if (linkAttempted &&
+            isAlreadyExistsError(error) &&
+            quarantineSidecarMatches(finalPath, `${content}\n`)) {
+            // An earlier replay installed this exact record.
+        }
+        else {
+            writeJournalRecoveryEvent("journal_quarantine_write_failed", "JournalQuarantineWriteError", companyUid);
+            throw error;
+        }
+    }
+    const eventKey = `${slug}:${record.sequence}:${digest}:${companyUid ?? ""}`;
+    if (!loggedQuarantinedJournalRecords.has(eventKey)) {
+        loggedQuarantinedJournalRecords.add(eventKey);
+        writeJournalRecoveryEvent("journal_record_quarantined", "JournalIncompleteUpsertError", companyUid);
+    }
+    const invalidPaths = new Set(invalid.map((entry) => entry.key));
+    const upserts = Object.fromEntries(Object.entries(delta.upserts).filter(([key]) => !invalidPaths.has(key)));
+    // A malformed upsert never authorizes deletion. Preserve any base row for
+    // that path, even if the same WAL record also listed it in `deletes`.
+    const deletes = delta.deletes.filter((key) => !invalidPaths.has(key));
+    return { ...delta, upserts, deletes };
+}
+function quarantineIncompleteAreaJournalRow(slug, key, entry) {
+    const delta = {
+        version: JOURNAL_VERSION_CURRENT,
+        upserts: { [key]: entry },
+        deletes: [],
+    };
+    quarantineIncompleteJournalUpserts(slug, undefined, { sequence: 0, type: JOURNAL_DELTA_RECORD_TYPE, payload: { source: "area-ledger-row" } }, delta, true);
+}
+function isAlreadyExistsError(error) {
+    return typeof error === "object" && error !== null && "code" in error &&
+        error.code === "EEXIST";
+}
+function quarantineSidecarMatches(filePath, expectedContent) {
+    try {
+        return fs.readFileSync(filePath, "utf8") === expectedContent;
+    }
+    catch {
+        return false;
+    }
+}
+function writeJournalRecoveryEvent(event, errorClass, companyUid) {
+    try {
+        process.stderr.write(`${JSON.stringify({
+            type: "error",
+            diagnostic: true,
+            event,
+            errorClass,
+            ...(companyUid === undefined ? {} : { companyUid }),
+        })}\n`);
+    }
+    catch {
+        // A diagnostic event must not break recovery or sync.
+        process.emitWarning(`journal recovery diagnostic could not be emitted (${errorClass})`);
+    }
+}
 function isJournal(value) {
     return typeof value === "object" && value !== null &&
         (value.version === "1" || value.version === "2") &&
@@ -2237,7 +2468,7 @@ function ensureJournalLocator(slug) {
  */
 export function repairJournalStateIfNeeded(slug, options = {}) {
     const safeSlug = sanitizeSlug(slug);
-    const storeOptions = journalStoreOptions(safeSlug, emptyJournal());
+    const storeOptions = journalStoreOptions(safeSlug, emptyJournal(), options.companyUid);
     if (options.maxWalBytesBeforeCompaction !== undefined) {
         storeOptions.maxWalBytesBeforeCompaction = options.maxWalBytesBeforeCompaction;
     }