@indigoai-us/hq-cloud 6.18.29 → 6.18.31

package/dist/bin/sync-runner-watch-loop.js

@@ -17,13 +17,13 @@ import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
 import { createIgnoreFilter } from "../ignore.js";
 import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
 import { recoverHqRootLocalState } from "../hq-root-recovery.js";
-import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
+import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
 import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
 import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
 import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
 import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
 import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
-import { resolveAreaLayoutAuthority } from "../sync/area-ledger-migration.js";
+import { readAreaLayoutState, resolveAreaLayoutAuthority } from "../sync/area-ledger-migration.js";
 import { AREA_AUTO_MIGRATION_SOURCE, createAreaAutoMigrationScheduler, openPersonalVaultAreaMigration, readPersonalVaultLayoutPhase, } from "../sync/area-ledger-auto-migration.js";
 import { emitCloudTelemetry } from "../telemetry-events.js";
 import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPressureCloudProperties, heapPressureStderrLine, heapRecycleCloudProperties, heapRecycleStderrLine, performHeapHardExit, nodeHeapReaders, nodeMajorGcObserver, resolveDeclaredHeapCeilingBytes, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, HEAP_SAMPLE_INTERVAL_ENV, HEAP_PRESSURE_FRACTION_ENV, HEAP_RECYCLE_ENABLED_ENV, HEAP_RECYCLE_LIVE_FRACTION_ENV, HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV, HEAP_RECYCLE_EMERGENCY_FRACTION_ENV, HEAP_RECYCLE_MIN_UPTIME_MS_ENV, HEAP_CHECKPOINT_MS_ENV, HEAP_PINNED_FRACTION_ENV, HEAP_PINNED_SUSTAIN_MS_ENV, HEAP_HARD_EXIT_FRACTION_ENV, HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV, HEAP_FOOTPRINT_RECYCLE_ENABLED_ENV, HEAP_FOOTPRINT_RECYCLE_MB_ENV, HEAP_FOOTPRINT_HARD_EXIT_MB_ENV, } from "./sync-runner-heap-telemetry.js";
@@ -1401,6 +1401,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
     };
     const monotonicNow = deps.monotonicNow ?? (() => performance.now());
     const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
+    const reportProtocolStatus = deps.reportProtocolStatus;
+    const reportLockWait = (holder) => {
+        const command = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(holder.command)
+            ? holder.command
+            : "unknown";
+        reportProtocolStatus?.({
+            state: "waiting-for-lock",
+            holder: { pid: holder.pid, command },
+        });
+    };
+    const reportRunnerRunning = () => {
+        reportProtocolStatus?.({ state: "running" });
+    };
     const qmdShutdownController = new AbortController();
     const runPassSupportsQmdLifecycle = deps.runPass === undefined || deps.runPassSupportsQmdLifecycle === true;
     const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
@@ -1435,6 +1448,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             skipPersonalEnv === "yes",
     };
     const hqRoot = parsed.hqRoot;
+    setAreaLedgerRecoveryHqRootForProcess(hqRoot);
     const serviceStartedAt = Date.now();
     // Delayed and daily: this must never be attached to the 15s poll cadence.
     // Stage 0 remains dry-run only; the standalone hq-backup-prune command is
@@ -1468,7 +1482,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         try {
             const handle = acquireOperationLock(hqRoot, "sync", {
                 timeoutSec: 0,
-                onWaitStart: () => undefined,
+                onWaitStart: (holder) => reportLockWait(holder),
                 deferToWaiters: true,
             });
             handle.release();
@@ -1716,6 +1730,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                 onWaitStart: () => undefined,
                 deferToWaiters: true,
             });
+            reportRunnerRunning();
             try {
                 telemetry?.mark("network");
                 if (lane === "slow") {
@@ -1750,7 +1765,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             }
         }
         try {
-            return await withOperationLock(hqRoot, "sync", async () => {
+            const outcome = await withOperationLock(hqRoot, "sync", async () => {
+                reportRunnerRunning();
                 telemetry?.mark("network");
                 if (lane === "slow") {
                     processOwnsOperationLock = true;
@@ -1771,9 +1787,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
                         processOwnsOperationLock = false;
                     }
                 }
-            }, { timeoutSec: parsed.lockTimeoutSec, deferToWaiters: true });
+            }, {
+                timeoutSec: parsed.lockTimeoutSec,
+                deferToWaiters: true,
+                onWaitStart: (holder) => reportLockWait(holder),
+            });
+            reportRunnerRunning();
+            return outcome;
         }
         catch (err) {
+            reportRunnerRunning();
             if (err instanceof OperationLockedError) {
                 process.stderr.write(err.message + "\n");
                 return OPERATION_LOCKED_EXIT;
@@ -1782,6 +1805,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         }
     };
     const runTaskWithLock = async (task, workClass = "chore") => {
+        reportRunnerRunning();
         if (workClass === "realtime" &&
             passScheduler.handoffActive &&
             processOwnsOperationLock) {
@@ -1796,9 +1820,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
         try {
             const handle = acquireOperationLock(hqRoot, "sync", {
                 timeoutSec: 0,
-                onWaitStart: () => undefined,
+                onWaitStart: (holder) => reportLockWait(holder),
                 deferToWaiters: true,
             });
+            reportRunnerRunning();
             try {
                 return await task();
             }
@@ -1815,12 +1840,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             }
         }
         try {
-            return await withOperationLock(hqRoot, "sync", task, {
+            const outcome = await withOperationLock(hqRoot, "sync", async () => {
+                reportRunnerRunning();
+                return task();
+            }, {
                 timeoutSec: parsed.lockTimeoutSec,
                 deferToWaiters: true,
+                onWaitStart: (holder) => reportLockWait(holder),
             });
+            reportRunnerRunning();
+            return outcome;
         }
         catch (err) {
+            reportRunnerRunning();
             if (err instanceof OperationLockedError) {
                 process.stderr.write(err.message + "\n");
                 return OPERATION_LOCKED_EXIT;
@@ -3617,6 +3649,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
             catch (err) {
                 process.stderr.write(`hq-sync-runner: WARN local HQ recovery failed: ${describeError(err)}\n`);
             }
+            if (personalJournalInScope) {
+                const layout = readAreaLayoutState(getStateDir(), PERSONAL_VAULT_JOURNAL_SLUG);
+                const activeCommit = layout?.activeCommitWatermark ?? 0;
+                if (layout?.phase === "area-authoritative" &&
+                    (!layout.rollbackSafe || (layout.legacyMirrorWatermark ?? 0) !== activeCommit)) {
+                    const recoveryStartedAt = performance.now();
+                    const recovery = withOperationLockSync(hqRoot, "area-ledger-migration", () => recoverCachedAreaJournalMirrorIfNeeded(PERSONAL_VAULT_JOURNAL_SLUG), { mode: "exclusive", timeoutSec: 5, onWaitStart: () => undefined });
+                    if (recovery?.recovered) {
+                        process.stderr.write(`hq-sync-runner: recovered area-ledger legacy mirror before watch ` +
+                            `(rows=${recovery.rowCount}, durationMs=${Math.round(performance.now() - recoveryStartedAt)})\n`);
+                    }
+                }
+            }
             watcher.start();
             const reportWatcherReady = () => {
                 if (stopped || watcher === null)