@indigoai-us/hq-cloud 6.16.64 → 6.16.66-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/dist/bin/sync-runner-company.d.ts +14 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +42 -8
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-company.test.js +62 -7
  6. package/dist/bin/sync-runner-company.test.js.map +1 -1
  7. package/dist/bin/sync-runner-planning.d.ts +12 -3
  8. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  9. package/dist/bin/sync-runner-planning.js +65 -3
  10. package/dist/bin/sync-runner-planning.js.map +1 -1
  11. package/dist/bin/sync-runner-planning.test.js +172 -1
  12. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  13. package/dist/bin/sync-runner-watch-loop.d.ts +52 -3
  14. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  15. package/dist/bin/sync-runner-watch-loop.js +451 -167
  16. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  17. package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
  18. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  19. package/dist/bin/sync-runner-watch-routes.js +40 -5
  20. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-routes.test.js +47 -0
  22. package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
  23. package/dist/bin/sync-runner.d.ts +64 -6
  24. package/dist/bin/sync-runner.d.ts.map +1 -1
  25. package/dist/bin/sync-runner.js +48 -4
  26. package/dist/bin/sync-runner.js.map +1 -1
  27. package/dist/bin/sync-runner.test.js +1577 -169
  28. package/dist/bin/sync-runner.test.js.map +1 -1
  29. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  30. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  31. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  32. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  33. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  34. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  35. package/dist/cli/onepass-pull-planner.test.js +266 -0
  36. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  37. package/dist/cli/push-scan-exclusions.test.js +4 -4
  38. package/dist/cli/push-scan-exclusions.test.js.map +1 -1
  39. package/dist/cli/share.d.ts +31 -1
  40. package/dist/cli/share.d.ts.map +1 -1
  41. package/dist/cli/share.js +136 -6
  42. package/dist/cli/share.js.map +1 -1
  43. package/dist/cli/share.test.js +295 -5
  44. package/dist/cli/share.test.js.map +1 -1
  45. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  46. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  47. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  48. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  49. package/dist/cli/sync-scope.test.js +8 -0
  50. package/dist/cli/sync-scope.test.js.map +1 -1
  51. package/dist/cli/sync.d.ts +12 -0
  52. package/dist/cli/sync.d.ts.map +1 -1
  53. package/dist/cli/sync.js +302 -123
  54. package/dist/cli/sync.js.map +1 -1
  55. package/dist/cli/sync.test.js +255 -2
  56. package/dist/cli/sync.test.js.map +1 -1
  57. package/dist/hq-root-recovery.d.ts +30 -0
  58. package/dist/hq-root-recovery.d.ts.map +1 -0
  59. package/dist/hq-root-recovery.js +131 -0
  60. package/dist/hq-root-recovery.js.map +1 -0
  61. package/dist/hq-root-recovery.test.d.ts +2 -0
  62. package/dist/hq-root-recovery.test.d.ts.map +1 -0
  63. package/dist/hq-root-recovery.test.js +65 -0
  64. package/dist/hq-root-recovery.test.js.map +1 -0
  65. package/dist/journal-row-store.d.ts +40 -5
  66. package/dist/journal-row-store.d.ts.map +1 -1
  67. package/dist/journal-row-store.js +240 -21
  68. package/dist/journal-row-store.js.map +1 -1
  69. package/dist/journal-row-store.test.js +54 -0
  70. package/dist/journal-row-store.test.js.map +1 -1
  71. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  72. package/dist/journal-snapshot-v4.js +11 -6
  73. package/dist/journal-snapshot-v4.js.map +1 -1
  74. package/dist/journal-store.test.js +324 -7
  75. package/dist/journal-store.test.js.map +1 -1
  76. package/dist/journal.d.ts +51 -16
  77. package/dist/journal.d.ts.map +1 -1
  78. package/dist/journal.js +303 -79
  79. package/dist/journal.js.map +1 -1
  80. package/dist/lib/exit-codes.d.ts +15 -0
  81. package/dist/lib/exit-codes.d.ts.map +1 -1
  82. package/dist/lib/exit-codes.js +15 -0
  83. package/dist/lib/exit-codes.js.map +1 -1
  84. package/dist/object-io.d.ts +33 -1
  85. package/dist/object-io.d.ts.map +1 -1
  86. package/dist/object-io.js +145 -8
  87. package/dist/object-io.js.map +1 -1
  88. package/dist/object-io.test.js +248 -1
  89. package/dist/object-io.test.js.map +1 -1
  90. package/dist/personal-vault.d.ts.map +1 -1
  91. package/dist/personal-vault.js +10 -3
  92. package/dist/personal-vault.js.map +1 -1
  93. package/dist/personal-vault.test.js +35 -0
  94. package/dist/personal-vault.test.js.map +1 -1
  95. package/dist/remote-pull.d.ts.map +1 -1
  96. package/dist/remote-pull.js +9 -2
  97. package/dist/remote-pull.js.map +1 -1
  98. package/dist/remote-pull.test.js +59 -2
  99. package/dist/remote-pull.test.js.map +1 -1
  100. package/dist/s3.d.ts +1 -0
  101. package/dist/s3.d.ts.map +1 -1
  102. package/dist/s3.js +20 -2
  103. package/dist/s3.js.map +1 -1
  104. package/dist/s3.list-cost.test.d.ts +2 -0
  105. package/dist/s3.list-cost.test.d.ts.map +1 -0
  106. package/dist/s3.list-cost.test.js +51 -0
  107. package/dist/s3.list-cost.test.js.map +1 -0
  108. package/dist/sync/area-ledger.d.ts +27 -3
  109. package/dist/sync/area-ledger.d.ts.map +1 -1
  110. package/dist/sync/area-ledger.js +149 -39
  111. package/dist/sync/area-ledger.js.map +1 -1
  112. package/dist/sync/area-ledger.test.js +192 -5
  113. package/dist/sync/area-ledger.test.js.map +1 -1
  114. package/dist/sync/parallel-sync-coordinator.test.js +29 -0
  115. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  116. package/dist/sync/presign-denial-backoff-flag.d.ts +22 -0
  117. package/dist/sync/presign-denial-backoff-flag.d.ts.map +1 -0
  118. package/dist/sync/presign-denial-backoff-flag.js +75 -0
  119. package/dist/sync/presign-denial-backoff-flag.js.map +1 -0
  120. package/dist/sync/presign-denial-backoff-flag.test.d.ts +2 -0
  121. package/dist/sync/presign-denial-backoff-flag.test.d.ts.map +1 -0
  122. package/dist/sync/presign-denial-backoff-flag.test.js +103 -0
  123. package/dist/sync/presign-denial-backoff-flag.test.js.map +1 -0
  124. package/dist/sync/realtime-telemetry.d.ts +1 -1
  125. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  126. package/dist/sync/realtime-telemetry.js +14 -6
  127. package/dist/sync/realtime-telemetry.js.map +1 -1
  128. package/dist/sync/realtime-telemetry.test.js +136 -16
  129. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  130. package/dist/sync/remote-file-spool.d.ts +16 -0
  131. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  132. package/dist/sync/remote-file-spool.js +91 -0
  133. package/dist/sync/remote-file-spool.js.map +1 -0
  134. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  135. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  136. package/dist/sync/remote-file-spool.test.js +77 -0
  137. package/dist/sync/remote-file-spool.test.js.map +1 -0
  138. package/dist/sync/state-store.d.ts +6 -0
  139. package/dist/sync/state-store.d.ts.map +1 -1
  140. package/dist/sync/state-store.js +14 -2
  141. package/dist/sync/state-store.js.map +1 -1
  142. package/dist/watch-coverage.d.ts +24 -0
  143. package/dist/watch-coverage.d.ts.map +1 -0
  144. package/dist/watch-coverage.js +46 -0
  145. package/dist/watch-coverage.js.map +1 -0
  146. package/dist/watch-coverage.test.d.ts +2 -0
  147. package/dist/watch-coverage.test.d.ts.map +1 -0
  148. package/dist/watch-coverage.test.js +56 -0
  149. package/dist/watch-coverage.test.js.map +1 -0
  150. package/dist/watcher.d.ts +31 -1
  151. package/dist/watcher.d.ts.map +1 -1
  152. package/dist/watcher.js +172 -0
  153. package/dist/watcher.js.map +1 -1
  154. package/dist/watcher.test.js +78 -0
  155. package/dist/watcher.test.js.map +1 -1
  156. package/package.json +3 -1
@@ -10,11 +10,12 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
10
10
  import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
11
11
  import { describeError } from "../lib/describe-error.js";
12
12
  import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
13
- import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
13
+ import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
14
14
  import { getOrCreateMachineId } from "../lib/machine-id.js";
15
15
  import { localPathForVaultKey } from "../local-path-codec.js";
16
16
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
17
- import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, readJournal, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
17
+ import { recoverHqRootLocalState } from "../hq-root-recovery.js";
18
+ import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
18
19
  import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
19
20
  import { NoopPushReceiver, } from "../sync/push-receiver.js";
20
21
  import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
@@ -24,12 +25,226 @@ import { AREA_AUTO_MIGRATION_SOURCE, createAreaAutoMigrationScheduler, openPerso
24
25
  import { emitCloudTelemetry } from "../telemetry-events.js";
25
26
  import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPressureCloudProperties, heapPressureStderrLine, heapRecycleCloudProperties, heapRecycleStderrLine, performHeapHardExit, nodeHeapReaders, nodeMajorGcObserver, resolveDeclaredHeapCeilingBytes, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, HEAP_SAMPLE_INTERVAL_ENV, HEAP_PRESSURE_FRACTION_ENV, HEAP_RECYCLE_ENABLED_ENV, HEAP_RECYCLE_LIVE_FRACTION_ENV, HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV, HEAP_RECYCLE_EMERGENCY_FRACTION_ENV, HEAP_RECYCLE_MIN_UPTIME_MS_ENV, HEAP_CHECKPOINT_MS_ENV, HEAP_PINNED_FRACTION_ENV, HEAP_PINNED_SUSTAIN_MS_ENV, HEAP_HARD_EXIT_FRACTION_ENV, HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV, HEAP_FOOTPRINT_RECYCLE_ENABLED_ENV, HEAP_FOOTPRINT_RECYCLE_MB_ENV, HEAP_FOOTPRINT_HARD_EXIT_MB_ENV, } from "./sync-runner-heap-telemetry.js";
26
27
  import { setRunnerEmitObserver } from "./sync-runner-events.js";
28
+ import { refreshPresignDenialBackoffFlagWithin } from "../sync/presign-denial-backoff-flag.js";
27
29
  import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
28
30
  import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr } from "../sync/realtime-telemetry.js";
29
31
  import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
30
32
  import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
31
33
  import { resolveSkipCompanies } from "./sync-runner-planning.js";
32
34
  const FULL_RECONCILE_EVERY_TICKS = 10;
35
+ /** A deferred full chore must get the next admission after this long. */
36
+ export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
37
+ /**
38
+ * One durable-state owner at a time for a full chore. Realtime work never runs
39
+ * beside that chore: it explicitly awaits `checkpoint()` after a completed
40
+ * durable unit, then resumes once the hand-off is finished. A separately
41
+ * tracked, explicitly opted-in watcher fast pass may use the pre-existing
42
+ * scoped fast-over-slow exception, but never while a chore owns durable state.
43
+ */
44
+ export class CooperativePassScheduler {
45
+ now;
46
+ starvationCapMs;
47
+ pending = [];
48
+ active = null;
49
+ // Scoped fast realtime work has the established in-process lock exception
50
+ // while a scoped slow realtime upload is in flight. Keep that slot separate
51
+ // from `active`: a chore still has exactly one durable-state owner.
52
+ activeFastRealtime = null;
53
+ stopped = false;
54
+ _handoffActive = false;
55
+ constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
56
+ this.now = now;
57
+ this.starvationCapMs = starvationCapMs;
58
+ }
59
+ get handoffActive() {
60
+ return this._handoffActive;
61
+ }
62
+ get hasActiveChore() {
63
+ return this.active?.workClass === "chore";
64
+ }
65
+ get hasActiveSlowRealtime() {
66
+ return this.active?.workClass === "realtime" && this.active.lane === "slow";
67
+ }
68
+ pendingLaneCount(lane) {
69
+ return this.pending.filter((entry) => entry.lane === lane).length;
70
+ }
71
+ submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
72
+ if (this.stopped)
73
+ return Promise.resolve(0);
74
+ const directEntry = {
75
+ task,
76
+ workClass,
77
+ lane,
78
+ allowFastRealtimeOvertake,
79
+ // An immediately admitted task cannot starve, so avoid sampling the
80
+ // cadence clock merely to stamp it. This preserves the loop's deadline
81
+ // accounting and keeps the timestamp meaningful only for real queues.
82
+ enqueuedAt: 0,
83
+ resolve: () => undefined,
84
+ reject: () => undefined,
85
+ };
86
+ if (this.canStartFastRealtime(directEntry)) {
87
+ return this.startFastRealtime(directEntry);
88
+ }
89
+ if (this.active === null &&
90
+ this.activeFastRealtime === null &&
91
+ this.pending.length === 0) {
92
+ return this.start(directEntry);
93
+ }
94
+ return new Promise((resolve, reject) => {
95
+ const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
96
+ this.pending.push(entry);
97
+ this.drain();
98
+ });
99
+ }
100
+ stop() {
101
+ this.stopped = true;
102
+ while (this.pending.length > 0)
103
+ this.pending.shift()?.resolve(0);
104
+ }
105
+ drain() {
106
+ if (this.stopped || this.pending.length === 0)
107
+ return;
108
+ if (this.active !== null) {
109
+ const fastRealtime = this.nextFastRealtime();
110
+ if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
111
+ this.pending.splice(this.pending.indexOf(fastRealtime), 1);
112
+ this.startFastRealtime(fastRealtime);
113
+ }
114
+ return;
115
+ }
116
+ if (this.activeFastRealtime !== null)
117
+ return;
118
+ const next = this.takeNext();
119
+ if (next)
120
+ this.start(next);
121
+ }
122
+ takeNext() {
123
+ const now = this.now();
124
+ const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
125
+ if (starvedChore >= 0)
126
+ return this.pending.splice(starvedChore, 1)[0];
127
+ // A receiver's bulk drain is still realtime work, but its slow lane is the
128
+ // progress boundary for that delivery. Preserve its established turn
129
+ // before refilling later fast realtime arrivals; this does not let a
130
+ // chore overtake any realtime work.
131
+ const realtime = this.nextRealtimeIndex();
132
+ if (realtime >= 0)
133
+ return this.pending.splice(realtime, 1)[0];
134
+ return this.pending.shift();
135
+ }
136
+ nextRealtimeIndex() {
137
+ const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
138
+ return slowRealtime >= 0
139
+ ? slowRealtime
140
+ : this.pending.findIndex((entry) => entry.workClass === "realtime");
141
+ }
142
+ nextFastRealtime() {
143
+ return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
144
+ }
145
+ hasStarvedChore() {
146
+ const now = this.now();
147
+ return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
148
+ }
149
+ canStartFastRealtime(entry) {
150
+ return (!this._handoffActive &&
151
+ this.activeFastRealtime === null &&
152
+ entry.workClass === "realtime" &&
153
+ entry.lane === "fast" &&
154
+ entry.allowFastRealtimeOvertake &&
155
+ this.hasActiveSlowRealtime &&
156
+ !this.hasStarvedChore());
157
+ }
158
+ start(entry) {
159
+ this.active = entry;
160
+ let current;
161
+ try {
162
+ current = this.execute(entry);
163
+ }
164
+ catch (error) {
165
+ current = Promise.reject(error);
166
+ }
167
+ void current.then((outcome) => {
168
+ entry.resolve(outcome);
169
+ this.complete(entry);
170
+ }, (error) => {
171
+ entry.reject(error);
172
+ this.complete(entry);
173
+ }).catch(() => undefined);
174
+ return current;
175
+ }
176
+ complete(entry) {
177
+ if (this.active === entry)
178
+ this.active = null;
179
+ this.drain();
180
+ }
181
+ startFastRealtime(entry) {
182
+ this.activeFastRealtime = entry;
183
+ let current;
184
+ try {
185
+ current = this.execute(entry);
186
+ }
187
+ catch (error) {
188
+ current = Promise.reject(error);
189
+ }
190
+ void current.then((outcome) => {
191
+ entry.resolve(outcome);
192
+ this.completeFastRealtime(entry);
193
+ }, (error) => {
194
+ entry.reject(error);
195
+ this.completeFastRealtime(entry);
196
+ });
197
+ return current;
198
+ }
199
+ completeFastRealtime(entry) {
200
+ if (this.activeFastRealtime === entry)
201
+ this.activeFastRealtime = null;
202
+ this.drain();
203
+ }
204
+ execute(entry) {
205
+ return entry.task({ checkpoint: () => this.checkpoint(entry) });
206
+ }
207
+ async checkpoint(entry) {
208
+ if (this.stopped ||
209
+ entry.workClass !== "chore" ||
210
+ this.active !== entry ||
211
+ !this.pending.some((pending) => pending.workClass === "realtime")) {
212
+ return false;
213
+ }
214
+ // `entry` is suspended at a caller-declared durable boundary. It remains
215
+ // the sole full-pass continuation and will resume after this bounded burst.
216
+ const deferredAt = this.now();
217
+ this.active = null;
218
+ this._handoffActive = true;
219
+ let handedOff = false;
220
+ try {
221
+ while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
222
+ const realtimeIndex = this.nextRealtimeIndex();
223
+ if (realtimeIndex < 0)
224
+ break;
225
+ const realtime = this.pending.splice(realtimeIndex, 1)[0];
226
+ handedOff = true;
227
+ this.active = realtime;
228
+ try {
229
+ realtime.resolve(await this.execute(realtime));
230
+ }
231
+ catch (error) {
232
+ realtime.reject(error);
233
+ }
234
+ finally {
235
+ if (this.active === realtime)
236
+ this.active = null;
237
+ }
238
+ }
239
+ }
240
+ finally {
241
+ this._handoffActive = false;
242
+ if (!this.stopped)
243
+ this.active = entry;
244
+ }
245
+ return handedOff;
246
+ }
247
+ }
33
248
  const MAX_FULL_RECONCILE_REASONS = 7;
34
249
  /**
35
250
  * Area storage is AUTO by default and only applies after durable cutover.
@@ -62,7 +277,7 @@ function emitFullReconcileDecision({ mode, reasons, triggers, pollTick, configur
62
277
  * is drained in bounded chunks across successive ticks instead of one giant
63
278
  * scoped pass. Size is not a full-reconcile signal — unknown scope is
64
279
  * (`overflowed` with no route hints, warm-up catch-up failure, first tick,
65
- * scheduled interval, bare wake). The crossover for a *chunk* is the number
280
+ * scheduled interval, bare wake, vanished HQ-root coverage). The crossover for a *chunk* is the number
66
281
  * of `--scope-path` arguments a targeted pass can carry without becoming a
67
282
  * de-facto whole-tree walk. `HQ_SYNC_EVENT_BATCH_LIMIT` overrides; 0/invalid
68
283
  * = default. Do not raise this to "fix" a large warm-up catch-up: that only
@@ -509,6 +724,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
509
724
  const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
510
725
  const monotonicNow = deps.monotonicNow ?? (() => performance.now());
511
726
  const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
727
+ const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
728
+ // Read the switch before the first pass, which is the largest presign user.
729
+ if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
730
+ await refreshPresignDenialBackoffFlagWithin({
731
+ endpoint: process.env.HQ_FLAGS_API_URL,
732
+ getToken: runtime.defaultGetAccessToken,
733
+ });
734
+ }
512
735
  const pollIdx = argv.indexOf("--poll-remote-ms");
513
736
  // An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
514
737
  // one). When it is omitted, the interval is resolved fresh each cycle from
@@ -519,7 +742,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
519
742
  const eventPush = argv.includes("--event-push");
520
743
  const fullReconcileIntervalMs = resolveFullReconcileMs();
521
744
  const companiesMode = argv.includes("--companies");
522
- const companyMode = argv.includes("--company");
745
+ const companyIdx = argv.indexOf("--company");
746
+ const companyMode = companyIdx >= 0;
747
+ // The tenant a single-company run owns. Carried into route selection so a
748
+ // watcher or receiver event under another company cannot queue a pass for it.
749
+ const companySlug = companyMode ? argv[companyIdx + 1] : undefined;
523
750
  const personalMode = argv.includes("--personal");
524
751
  // Keep this at the watcher ingress rather than importing the runner's
525
752
  // resolver: sync-runner imports this loop, and a reverse value import would
@@ -528,6 +755,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
528
755
  const watchRouteSelection = {
529
756
  companies: companiesMode,
530
757
  company: companyMode,
758
+ ...(companySlug !== undefined ? { companySlug } : {}),
531
759
  personal: personalMode,
532
760
  skipPersonal: parsed.skipPersonal === true ||
533
761
  skipPersonalEnv === "1" ||
@@ -672,7 +900,27 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
672
900
  resolveNoOperationLockUsers = resolve;
673
901
  });
674
902
  };
675
- const runPassWithLock = async (passArgvForRun, prepare, lane = "slow") => {
903
+ const passScheduler = new CooperativePassScheduler(monotonicNow);
904
+ const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
905
+ const reportedAppliedUpserts = new Set();
906
+ const recordDurableAppliedUpserts = (upserts) => {
907
+ for (const applied of upserts) {
908
+ const key = `${applied.relativePath}\u0000${applied.contentHash}`;
909
+ if (reportedAppliedUpserts.has(key))
910
+ continue;
911
+ reportedAppliedUpserts.add(key);
912
+ rememberAppliedUpsert(applied.relativePath, applied.contentHash);
913
+ }
914
+ };
915
+ const invokePass = () => {
916
+ if (!runPassSupportsCooperativeCheckpoint)
917
+ return runPass(passArgvForRun);
918
+ // This callback is emitted by the production runner after journal
919
+ // repair/seed and before fanout. It starts ingress early, but never
920
+ // admits a second durable writer: queued realtime still waits for the
921
+ // existing scheduler's cooperative hand-off checkpoint.
922
+ return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
923
+ };
676
924
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
677
925
  // admitting a pass and immediately after it returns. A pass that runs a long
678
926
  // synchronous stretch starves the interval timer, so these boundary
@@ -684,9 +932,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
684
932
  heapCheckpoint?.();
685
933
  if (typeof outcome === "number")
686
934
  return;
687
- for (const applied of outcome.result?.appliedUpserts ?? []) {
688
- rememberAppliedUpsert(applied.relativePath, applied.contentHash);
689
- }
935
+ recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
690
936
  };
691
937
  // Pre-admit boundary observation.
692
938
  heapCheckpoint?.();
@@ -695,13 +941,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
695
941
  // its keyed journal delta and per-object mutation are independent from the
696
942
  // slow pass's bulk paths, and the slow pass already owns the root lock.
697
943
  // Never bypass the lock when no slow pass is active.
944
+ if (
945
+ // A full chore is paused at a caller-declared durable checkpoint. Keep
946
+ // the root lock held by this process, but lend it to exactly one
947
+ // realtime pass; the suspended chore cannot mutate until this returns.
948
+ workClass === "realtime" &&
949
+ passScheduler.handoffActive &&
950
+ processOwnsOperationLock) {
951
+ acquireOperationLockUser();
952
+ try {
953
+ prepare?.();
954
+ const outcome = await invokePass();
955
+ recordAppliedUpserts(outcome);
956
+ return outcome;
957
+ }
958
+ finally {
959
+ releaseOperationLockUser();
960
+ }
961
+ }
698
962
  if (lane === "fast" &&
699
963
  processOwnsOperationLock &&
700
964
  coordinatorWholeMutationChores === 0) {
701
965
  acquireOperationLockUser();
702
966
  try {
703
967
  prepare?.();
704
- const outcome = await runPass(passArgvForRun);
968
+ const outcome = await invokePass();
705
969
  recordAppliedUpserts(outcome);
706
970
  return outcome;
707
971
  }
@@ -721,7 +985,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
721
985
  acquireOperationLockUser();
722
986
  }
723
987
  prepare?.();
724
- const outcome = await runPass(passArgvForRun);
988
+ const outcome = await invokePass();
725
989
  recordAppliedUpserts(outcome);
726
990
  return outcome;
727
991
  }
@@ -755,7 +1019,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
755
1019
  }
756
1020
  try {
757
1021
  prepare?.();
758
- const outcome = await runPass(passArgvForRun);
1022
+ const outcome = await invokePass();
759
1023
  recordAppliedUpserts(outcome);
760
1024
  return outcome;
761
1025
  }
@@ -778,7 +1042,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
778
1042
  throw err;
779
1043
  }
780
1044
  };
781
- const runTaskWithLock = async (task) => {
1045
+ const runTaskWithLock = async (task, workClass = "chore") => {
1046
+ if (workClass === "realtime" &&
1047
+ passScheduler.handoffActive &&
1048
+ processOwnsOperationLock) {
1049
+ acquireOperationLockUser();
1050
+ try {
1051
+ return await task();
1052
+ }
1053
+ finally {
1054
+ releaseOperationLockUser();
1055
+ }
1056
+ }
782
1057
  try {
783
1058
  const handle = acquireOperationLock(hqRoot, "sync", {
784
1059
  timeoutSec: 0,
@@ -872,102 +1147,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
872
1147
  heapCheckpoint?.();
873
1148
  }
874
1149
  };
875
- let activeSlowPass = null;
876
- let activeFastPass = null;
877
- const pendingSlowPasses = [];
878
- const pendingFastPasses = [];
879
1150
  const resolveStoppedQueue = () => {
880
- for (const queue of [pendingFastPasses, pendingSlowPasses]) {
881
- while (queue.length > 0)
882
- queue.shift()?.resolve(0);
883
- }
884
- };
885
- const drainQueuedPasses = () => {
886
- if (stopped) {
887
- resolveStoppedQueue();
888
- return;
889
- }
890
- // A queued slow pass is the bulk-drain progress boundary. Admit it before
891
- // refilling the realtime queue; otherwise a steady stream of fast work
892
- // perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
893
- // cannot issue another receive.
894
- if (activeSlowPass === null &&
895
- activeFastPass === null &&
896
- pendingSlowPasses.length > 0) {
897
- const slow = pendingSlowPasses.shift();
898
- if (slow) {
899
- const current = startGuardedTask(slow.task, "slow");
900
- void current.then(slow.resolve, slow.reject);
901
- }
902
- }
903
- if (activeFastPass === null) {
904
- const fast = pendingFastPasses.shift();
905
- if (fast) {
906
- const current = startGuardedTask(fast.task, "fast");
907
- void current.then(fast.resolve, fast.reject);
908
- }
909
- }
910
- // A fast pass that acquired the operation lock normally (rather than
911
- // bypassing an in-process slow holder) still owns the cross-process
912
- // boundary. Slow work must queue behind it; only fast work is allowed to
913
- // overlap an already-lock-owning slow pass.
914
- if (activeSlowPass === null && activeFastPass === null) {
915
- const slow = pendingSlowPasses.shift();
916
- if (slow) {
917
- const current = startGuardedTask(slow.task, "slow");
918
- void current.then(slow.resolve, slow.reject);
919
- }
920
- }
921
- };
922
- const startGuardedTask = (task, lane) => {
923
- // Claim the lane before invoking `task`: a task can synchronously activate
924
- // a watcher/scheduler callback before returning its promise. Without this
925
- // placeholder that re-entrant callback sees an idle lane and starts a
926
- // second lock contender.
927
- const placeholder = Promise.resolve(0);
928
- if (lane === "fast")
929
- activeFastPass = placeholder;
930
- else
931
- activeSlowPass = placeholder;
932
- let current;
933
- try {
934
- current = stopped ? placeholder : task();
935
- }
936
- catch (err) {
937
- current = Promise.reject(err);
938
- }
939
- if (lane === "fast")
940
- activeFastPass = current;
941
- else
942
- activeSlowPass = current;
943
- void current
944
- .finally(() => {
945
- if (lane === "fast" && activeFastPass === current) {
946
- activeFastPass = null;
947
- drainQueuedPasses();
948
- }
949
- else if (lane === "slow" && activeSlowPass === current) {
950
- activeSlowPass = null;
951
- drainQueuedPasses();
952
- }
953
- })
954
- .catch(() => undefined);
955
- return current;
956
- };
957
- const runGuardedTask = (task, lane = "slow") => {
958
- const active = lane === "fast"
959
- ? activeFastPass
960
- : activeSlowPass ?? activeFastPass;
961
- const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
962
- if (active === null && queued.length === 0) {
963
- return startGuardedTask(task, lane);
964
- }
965
- return new Promise((resolve, reject) => {
966
- queued.push({ task, lane, resolve, reject });
967
- drainQueuedPasses();
968
- });
1151
+ passScheduler.stop();
969
1152
  };
970
- const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
1153
+ const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) =>
1154
+ // Lanes still choose the pass execution profile, but no longer decide
1155
+ // chore ownership. Realtime normally waits for a chore hand-off; the
1156
+ // watcher fast path explicitly opts into its established scoped exception.
1157
+ passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake);
1158
+ const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake);
971
1159
  /*
972
1160
  * Area storage and the parallel coordinator default to AUTO. AUTO turns
973
1161
  * them on after durable personal-vault cutover; explicit off is the kill
@@ -1159,6 +1347,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1159
1347
  });
1160
1348
  },
1161
1349
  onCutOver: () => {
1350
+ releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
1162
1351
  parallelCoordinatorEnabled = refreshCoordinatorEnabled();
1163
1352
  if (parallelCoordinatorEnabled && coordinator === null) {
1164
1353
  coordinator = createCoordinator();
@@ -1167,6 +1356,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1167
1356
  })
1168
1357
  : { onTick() { } };
1169
1358
  let watcher = null;
1359
+ let rootCoverageStale = false;
1170
1360
  let eventPushSurfacesActivated = false;
1171
1361
  let watcherUnavailableWarned = false;
1172
1362
  let watcherRetryAt = null;
@@ -1240,11 +1430,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1240
1430
  `watched-paths=${watchedPaths ?? "unknown"}, ` +
1241
1431
  `dropped-untracked-deletes=${watcher.untrackedDeletesDropped?.() ?? "unknown"}, ` +
1242
1432
  `drained-untracked-deletes=${drainedUntrackedDeletes})\n`);
1433
+ const probe = watcher.probeCoverage?.();
1434
+ if (probe && !probe.ok) {
1435
+ process.stderr.write("hq-sync-runner: WARN event-push watcher coverage stale; " +
1436
+ `reason=${probe.reason} stale-watches=${probe.staleWatches} ` +
1437
+ `watched-paths=${watchedPaths ?? "unknown"}; restoring from vault, not pushing deletes\n`);
1438
+ try {
1439
+ recoverHqRootLocalState(hqRoot);
1440
+ }
1441
+ catch (err) {
1442
+ process.stderr.write(`hq-sync-runner: WARN local HQ recovery failed: ${describeError(err)}\n`);
1443
+ }
1444
+ try {
1445
+ watcher.rebuildCoverage?.();
1446
+ }
1447
+ catch (err) {
1448
+ process.stderr.write(`hq-sync-runner: WARN watcher coverage rebuild failed: ${describeError(err)}\n`);
1449
+ }
1450
+ rootCoverageStale = true;
1451
+ interruptWait();
1452
+ }
1243
1453
  };
1244
1454
  /** Vanished never-journaled file deletes removed from batches at drain time. */
1245
1455
  let drainedUntrackedDeletes = 0;
1246
1456
  const pendingWatcherPaths = new Map();
1247
1457
  const pendingWatcherChanges = new Map();
1458
+ // A warm-up scan observes filesystem state over time. Keep an ingress
1459
+ // sequence so its older observations cannot overwrite a live watcher event
1460
+ // that arrived while the scan was yielding.
1461
+ const pendingWatcherChangeSequences = new Map();
1462
+ let nextPendingWatcherChangeSequence = 0;
1248
1463
  let pendingWatcherOriginalBatch = null;
1249
1464
  // Every coalesced watcher batch gets one durable in-process generation.
1250
1465
  // A wake can cause the scheduler to revisit a drained batch, but an event
@@ -1268,11 +1483,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1268
1483
  const paths = new Map();
1269
1484
  const changes = new Map();
1270
1485
  for (const [absolutePath, relativePath] of batch.paths.entries()) {
1271
- const route = routeChangeToTarget(relativePath);
1486
+ const change = batch.changes?.get(absolutePath);
1487
+ const route = routeChangeToTarget(relativePath, change?.kind);
1272
1488
  if (!route || !isWatchRouteSelected(route, watchRouteSelection))
1273
1489
  continue;
1274
1490
  paths.set(absolutePath, relativePath);
1275
- const change = batch.changes?.get(absolutePath);
1276
1491
  if (change)
1277
1492
  changes.set(absolutePath, change);
1278
1493
  }
@@ -1280,7 +1495,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1280
1495
  return null;
1281
1496
  return { ...batch, paths, changes };
1282
1497
  };
1283
- const addPendingWatcherChange = (changedRelPath, batch) => {
1498
+ const addPendingWatcherChange = (changedRelPath, batch, options) => {
1284
1499
  if (batch) {
1285
1500
  const selectedBatch = selectWatchBatch(batch);
1286
1501
  if (!selectedBatch)
@@ -1293,20 +1508,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1293
1508
  return false;
1294
1509
  }
1295
1510
  }
1296
- pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
1297
1511
  if (batch) {
1512
+ const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
1513
+ const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
1514
+ return previousSequence === undefined ||
1515
+ options?.preserveChangesAfter === undefined ||
1516
+ previousSequence <= options.preserveChangesAfter;
1517
+ });
1518
+ if (entries.length === 0)
1519
+ return false;
1520
+ pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
1298
1521
  pendingWatcherOriginalBatch =
1299
1522
  pendingWatcherPaths.size === 0 &&
1300
1523
  !pendingWatcherBareChange &&
1301
1524
  !pendingWatcherOverflowed &&
1302
- !batch.overflowed
1525
+ !batch.overflowed &&
1526
+ entries.length === batch.paths.size
1303
1527
  ? batch
1304
1528
  : null;
1305
- for (const [absolutePath, relativePath] of batch.paths.entries()) {
1529
+ for (const [absolutePath, relativePath] of entries) {
1306
1530
  pendingWatcherPaths.set(absolutePath, relativePath);
1307
1531
  const change = batch.changes?.get(absolutePath);
1308
1532
  if (change)
1309
1533
  pendingWatcherChanges.set(absolutePath, change);
1534
+ pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
1310
1535
  }
1311
1536
  if (batch.overflowed) {
1312
1537
  pendingWatcherOverflowed = true;
@@ -1323,6 +1548,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1323
1548
  }
1324
1549
  return true;
1325
1550
  }
1551
+ pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
1326
1552
  if (changedRelPath) {
1327
1553
  pendingWatcherOriginalBatch = null;
1328
1554
  // Codec boundary: `changedRelPath` is a canonical vault key (the
@@ -1331,6 +1557,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1331
1557
  // point at the impossible colon path on Windows.
1332
1558
  const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
1333
1559
  pendingWatcherPaths.set(absolutePath, changedRelPath);
1560
+ pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
1334
1561
  // A bare path signal carries no delete intent; drop any change record
1335
1562
  // a prior batch left for this path so the stale intent cannot leak
1336
1563
  // into the coalesced batch (#227).
@@ -1372,6 +1599,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1372
1599
  const generation = pendingWatcherBatchGeneration;
1373
1600
  pendingWatcherPaths.clear();
1374
1601
  pendingWatcherChanges.clear();
1602
+ pendingWatcherChangeSequences.clear();
1375
1603
  pendingWatcherOriginalBatch = null;
1376
1604
  pendingWatcherBareChange = false;
1377
1605
  pendingWatcherWarmupCatchup = false;
@@ -1390,6 +1618,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1390
1618
  pendingWatcherOriginalBatch = null;
1391
1619
  for (const [absolutePath, relativePath] of paths.entries()) {
1392
1620
  pendingWatcherPaths.set(absolutePath, relativePath);
1621
+ pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
1393
1622
  }
1394
1623
  };
1395
1624
  /**
@@ -1470,8 +1699,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1470
1699
  }
1471
1700
  return null;
1472
1701
  };
1473
- const journalCoordinates = (relativePath) => {
1474
- const route = routeChangeToTarget(relativePath);
1702
+ const journalCoordinates = (relativePath, kind) => {
1703
+ const route = routeChangeToTarget(relativePath, kind);
1475
1704
  if (!route || !isWatchRouteSelected(route, watchRouteSelection))
1476
1705
  return null;
1477
1706
  if (route.kind === "personal") {
@@ -1506,7 +1735,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1506
1735
  declaredSize >= 0) {
1507
1736
  return declaredSize;
1508
1737
  }
1509
- const size = journalRows(readJournal(coordinates.slug)).get(coordinates.key)?.size;
1738
+ const size = journalRows(readJournalScoped(coordinates.slug, [coordinates.key]))
1739
+ .get(coordinates.key)?.size;
1510
1740
  return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
1511
1741
  };
1512
1742
  /**
@@ -1553,8 +1783,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1553
1783
  return session.get(coordinates.key) === undefined;
1554
1784
  };
1555
1785
  const captureLocalDeleteSnapshots = (relativePath, kind) => {
1556
- const coordinates = journalCoordinates(relativePath);
1557
- const route = routeChangeToTarget(relativePath);
1786
+ const coordinates = journalCoordinates(relativePath, kind);
1787
+ const route = routeChangeToTarget(relativePath, kind);
1558
1788
  if (!coordinates || !route)
1559
1789
  return [];
1560
1790
  // Desktop workspace Off toggles are authoritative before journal I/O.
@@ -1675,8 +1905,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1675
1905
  * it scopes a push or pull for the affected batch.
1676
1906
  */
1677
1907
  const captureQueuedLocalDeleteSnapshots = async (relativePath, kind, isCurrent) => {
1678
- const coordinates = journalCoordinates(relativePath);
1679
- const route = routeChangeToTarget(relativePath);
1908
+ const coordinates = journalCoordinates(relativePath, kind);
1909
+ const route = routeChangeToTarget(relativePath, kind);
1680
1910
  if (!coordinates || !route)
1681
1911
  return [];
1682
1912
  if (route.kind === "company" &&
@@ -1770,8 +2000,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1770
2000
  }
1771
2001
  return snapshots;
1772
2002
  };
1773
- const warmupJournalPaths = () => {
1774
- const paths = [];
2003
+ const warmupJournalPaths = function* () {
1775
2004
  const journals = listJournals();
1776
2005
  const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
1777
2006
  for (const { slug, journal } of journals) {
@@ -1804,10 +2033,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1804
2033
  // ~12.8k of them per runner start after a transcript prune.
1805
2034
  if (route.kind === "personal" && isPersonalVaultDeletionExcluded(key))
1806
2035
  continue;
1807
- paths.push(route.kind === "personal" ? key : `companies/${route.slug}/${key}`);
2036
+ yield route.kind === "personal" ? key : `companies/${route.slug}/${key}`;
1808
2037
  }
1809
2038
  }
1810
- return paths;
1811
2039
  };
1812
2040
  const publishedContentHashLookup = (relativePath) => {
1813
2041
  const coordinates = journalCoordinates(relativePath);
@@ -1820,13 +2048,34 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1820
2048
  return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
1821
2049
  };
1822
2050
  const prepareWatcherChanges = (changes) => {
2051
+ const keysByJournal = new Map();
2052
+ const rememberKey = (slug, key) => {
2053
+ const keys = keysByJournal.get(slug) ?? new Set();
2054
+ keys.add(key);
2055
+ keysByJournal.set(slug, keys);
2056
+ };
2057
+ for (const [absolutePath, change] of changes.entries()) {
2058
+ if (change.kind === "unlink" || change.kind === "unlinkDir") {
2059
+ for (const snapshot of change.deleteSnapshots ?? []) {
2060
+ rememberKey(snapshot.journalSlug, snapshot.journalPath);
2061
+ }
2062
+ continue;
2063
+ }
2064
+ const relativePath = path
2065
+ .relative(hqRoot, absolutePath)
2066
+ .split(path.sep)
2067
+ .join("/");
2068
+ const coordinates = journalCoordinates(relativePath);
2069
+ if (coordinates)
2070
+ rememberKey(coordinates.slug, coordinates.key);
2071
+ }
1823
2072
  const journals = new Map();
1824
2073
  const dirty = new Set();
1825
2074
  const journalFor = (slug) => {
1826
2075
  const loaded = journals.get(slug);
1827
2076
  if (loaded)
1828
2077
  return loaded;
1829
- const journal = readJournal(slug);
2078
+ const journal = readJournalScoped(slug, [...(keysByJournal.get(slug) ?? [])]);
1830
2079
  journals.set(slug, journal);
1831
2080
  return journal;
1832
2081
  };
@@ -1884,7 +2133,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1884
2133
  const groupBatchByRoute = (batch) => {
1885
2134
  const grouped = new Map();
1886
2135
  for (const [absolutePath, relativePath] of batch.paths.entries()) {
1887
- const route = routeChangeToTarget(relativePath);
2136
+ const change = batch.changes?.get(absolutePath);
2137
+ const route = routeChangeToTarget(relativePath, change?.kind);
1888
2138
  if (!route)
1889
2139
  continue;
1890
2140
  const key = routeKey(route);
@@ -1897,7 +2147,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1897
2147
  deleteScopeRoots: [],
1898
2148
  };
1899
2149
  existing.paths.set(absolutePath, relativePath);
1900
- const change = batch.changes?.get(absolutePath);
1901
2150
  if (change)
1902
2151
  existing.changes.set(absolutePath, change);
1903
2152
  existing.relPaths = [...new Set([...existing.paths.values()])];
@@ -1981,7 +2230,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1981
2230
  }
1982
2231
  return false;
1983
2232
  };
1984
- const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
2233
+ const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
1985
2234
  if (batch)
1986
2235
  await resolveDeleteSnapshotCaptures(batch);
1987
2236
  const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
@@ -2010,11 +2259,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2010
2259
  const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
2011
2260
  return runGuarded(scopedArgv, admittedChanges.size > 0
2012
2261
  ? () => prepareWatcherChanges(admittedChanges)
2013
- : undefined, lane);
2262
+ : undefined, lane, "realtime", allowFastRealtimeOvertake);
2014
2263
  })
2015
2264
  : { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
2016
2265
  ? () => prepareWatcherChanges(group.changes)
2017
- : undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
2266
+ : undefined, lane, "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
2018
2267
  const result = execution.result;
2019
2268
  if (execution.accepted && passExitCode(result) === 0) {
2020
2269
  const refusedPaths = new Set(typeof result === "number"
@@ -2084,8 +2333,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2084
2333
  ? "fast"
2085
2334
  : "slow";
2086
2335
  const execution = parallelCoordinatorEnabled
2087
- ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
2088
- : { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
2336
+ ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))
2337
+ : { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
2089
2338
  const exit = execution.accepted ? passExitCode(execution.result) : 1;
2090
2339
  if (exit !== 0 && worstExit === 0)
2091
2340
  worstExit = exit;
@@ -2290,13 +2539,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2290
2539
  return;
2291
2540
  }
2292
2541
  if (lanes?.fast &&
2293
- activeSlowPass !== null) {
2542
+ (passScheduler.hasActiveChore || passScheduler.hasActiveSlowRealtime)) {
2294
2543
  const generation = ++nextWatcherBatchGeneration;
2295
2544
  // Fast dispatch bypasses the pending-batch ingress, so preserve both
2296
2545
  // of its contracts here: only selected routes may run, and V2 sees
2297
2546
  // the same dirty signal as the compatibility drain.
2298
2547
  realtimeState.scheduler?.signal("watcher");
2299
- void runScopedDrain(lanes.fast, generation, true).then((exit) => {
2548
+ void runScopedDrain(lanes.fast, generation, true, true).then((exit) => {
2300
2549
  if (exit === 0)
2301
2550
  return;
2302
2551
  restorePendingWatcherPaths(lanes.fast.paths, generation);
@@ -2320,6 +2569,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2320
2569
  interruptWait();
2321
2570
  }
2322
2571
  });
2572
+ try {
2573
+ recoverHqRootLocalState(hqRoot);
2574
+ }
2575
+ catch (err) {
2576
+ process.stderr.write(`hq-sync-runner: WARN local HQ recovery failed: ${describeError(err)}\n`);
2577
+ }
2323
2578
  watcher.start();
2324
2579
  const reportWatcherReady = () => {
2325
2580
  if (stopped || watcher === null)
@@ -2345,29 +2600,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2345
2600
  watcherRediscoveryRetries = 0;
2346
2601
  process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
2347
2602
  const startedAt = Date.now();
2348
- try {
2349
- const catchup = watcher.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
2350
- if (!catchup)
2351
- return;
2352
- const queued = addPendingWatcherChange(undefined, catchup.batch);
2353
- pendingWatcherWarmupCatchup ||= queued;
2354
- process.stderr.write(JSON.stringify({
2355
- type: "watcher-warmup-catchup",
2356
- reason: "watch-set-ready",
2357
- pathsFound: catchup.batch.paths.size,
2358
- scannedPaths: catchup.scannedPaths,
2359
- elapsedMs: Date.now() - startedAt,
2360
- queued,
2361
- }) + "\n");
2362
- if (queued)
2603
+ const liveChangeSequenceAtWarmupStart = nextPendingWatcherChangeSequence;
2604
+ void (async () => {
2605
+ try {
2606
+ const catchup = watcher?.collectWarmupCatchupAsync
2607
+ ? await watcher.collectWarmupCatchupAsync(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths())
2608
+ : watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
2609
+ if (!catchup || catchup.cancelled || stopped || watcher === null)
2610
+ return;
2611
+ const queued = addPendingWatcherChange(undefined, catchup.batch, {
2612
+ preserveChangesAfter: liveChangeSequenceAtWarmupStart,
2613
+ });
2614
+ pendingWatcherWarmupCatchup ||= queued;
2615
+ process.stderr.write(JSON.stringify({
2616
+ type: "watcher-warmup-catchup",
2617
+ reason: "watch-set-ready",
2618
+ pathsFound: catchup.batch.paths.size,
2619
+ scannedPaths: catchup.scannedPaths,
2620
+ elapsedMs: Date.now() - startedAt,
2621
+ queued,
2622
+ }) + "\n");
2623
+ if (queued)
2624
+ interruptWait();
2625
+ }
2626
+ catch (err) {
2627
+ process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
2628
+ // A scan failure must not silently extend the warm-up blind spot.
2629
+ pendingWatcherWarmupCatchupFailed = true;
2363
2630
  interruptWait();
2364
- }
2365
- catch (err) {
2366
- process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
2367
- // A scan failure must not silently extend the warm-up blind spot.
2368
- pendingWatcherWarmupCatchupFailed = true;
2369
- interruptWait();
2370
- }
2631
+ }
2632
+ })();
2371
2633
  };
2372
2634
  if (watcher.onReady)
2373
2635
  watcher.onReady(reportWatcherReady);
@@ -2415,7 +2677,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2415
2677
  // Submit every route in this lane before awaiting any result. In
2416
2678
  // particular, a later route's fast receipt must never wait for an
2417
2679
  // earlier route's slow pull to finish.
2418
- const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
2680
+ const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))));
2419
2681
  for (const execution of completed) {
2420
2682
  if (!execution.accepted || passExitCode(execution.result) !== 0) {
2421
2683
  throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
@@ -2469,7 +2731,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2469
2731
  }
2470
2732
  for (const { route, paths } of pathsByRoute.values()) {
2471
2733
  const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
2472
- const result = await runGuarded(targetedArgv, undefined, lane);
2734
+ const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
2473
2735
  if (passExitCode(result) !== 0) {
2474
2736
  throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
2475
2737
  }
@@ -2517,14 +2779,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2517
2779
  const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
2518
2780
  await drain();
2519
2781
  return 0;
2520
- })));
2782
+ }, "realtime"), "slow", "realtime"));
2521
2783
  outcome = execution.accepted ? execution.result : 1;
2522
2784
  }
2523
2785
  else {
2524
2786
  outcome = await runGuardedTask(() => runTaskWithLock(async () => {
2525
2787
  await drain();
2526
2788
  return 0;
2527
- }));
2789
+ }, "realtime"), "slow", "realtime");
2528
2790
  }
2529
2791
  }
2530
2792
  catch (err) {
@@ -2720,8 +2982,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2720
2982
  targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
2721
2983
  targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
2722
2984
  cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
2723
- pendingSlowPasses: () => pendingSlowPasses.length,
2724
- pendingFastPasses: () => pendingFastPasses.length,
2985
+ pendingSlowPasses: () => passScheduler.pendingLaneCount("slow"),
2986
+ pendingFastPasses: () => passScheduler.pendingLaneCount("fast"),
2725
2987
  },
2726
2988
  eventSyncSizes: () => eventSync?.censusSizes?.(),
2727
2989
  watcherSizes: () => watcher?.censusSizes?.(),
@@ -2909,6 +3171,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2909
3171
  const watcherWarmupCatchup = pending.warmupCatchup;
2910
3172
  const watcherWarmupCatchupFailed = pending.warmupCatchupFailed;
2911
3173
  const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
3174
+ const coverageStale = rootCoverageStale;
3175
+ rootCoverageStale = false;
2912
3176
  const scheduledInterval = fullReconcileIntervalMs === undefined
2913
3177
  ? pollTick % FULL_RECONCILE_EVERY_TICKS === 0
2914
3178
  : lastFullReconcileAt !== null &&
@@ -2922,6 +3186,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2922
3186
  "watcher-warmup-catch-up": watcherWarmupCatchup,
2923
3187
  "watcher-warmup-catch-up-failed": watcherWarmupCatchupFailed,
2924
3188
  "scheduled-interval": scheduledInterval,
3189
+ "root-coverage-stale": coverageStale,
2925
3190
  };
2926
3191
  const reasons = Object.entries(triggers)
2927
3192
  .filter(([, matched]) => matched)
@@ -2931,7 +3196,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2931
3196
  bareWake ||
2932
3197
  overflowRoutesUnknown ||
2933
3198
  watcherWarmupCatchupFailed ||
2934
- scheduledInterval;
3199
+ scheduledInterval ||
3200
+ coverageStale;
2935
3201
  // Size is not unknown-scope. A fully-enumerated over-limit batch still
2936
3202
  // has every path listed, so drain it in bounded chunks and keep the
2937
3203
  // remainder queued. Skip this when a genuine full-reconcile trigger
@@ -3071,11 +3337,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3071
3337
  quarantinedDeleteRoots.delete(root);
3072
3338
  }
3073
3339
  // Delete snapshot capture needs a fresh keyed view while an episode is
3074
- // being assembled, but completed watch cycles do not. Releasing it here
3075
- // prevents the watcher from retaining another full-vault projection next
3076
- // to the targeted-pull and area-ledger views.
3340
+ // being assembled, but completed watch cycles do not. Area-authoritative
3341
+ // journals are different: their bridge owns the packed aggregate that a
3342
+ // later watcher callback reuses. Dropping that bridge at every poll
3343
+ // makes the next scoped callback synchronously re-pack the whole vault.
3344
+ // Keep it for this runner's lifetime; layout/phase changes still close
3345
+ // it in openAreaMigrationIfAuthoritative(), and final shutdown releases
3346
+ // every bridge below.
3077
3347
  releaseWatcherJournalStores();
3078
- releaseCachedAreaJournalMigrations();
3079
3348
  const exitCode = passExitCode(result);
3080
3349
  if (exitCode === runtime.authRequiredPassExit) {
3081
3350
  // The pass already emitted auth-error. Hand the sentinel to the
@@ -3110,8 +3379,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3110
3379
  : "") +
3111
3380
  "; staying alive, retrying next poll\n");
3112
3381
  }
3382
+ else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
3383
+ // HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
3384
+ // company directory names, which may not be memberships. The runner
3385
+ // has already emitted the resolution error; terminating here would
3386
+ // restart the outpost watcher every 60 seconds for each such route.
3387
+ // Keep the watcher alive while preserving the one-shot non-zero code.
3388
+ process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
3389
+ }
3113
3390
  else if (exitCode !== 0) {
3114
- return exitCode;
3391
+ // A scoped company-resolution or per-pass failure is isolated to that
3392
+ // pass. The error was emitted by the runner; terminating the daemon
3393
+ // recreates the startup blind window and strands newer watcher work.
3394
+ process.stderr.write(`hq-sync-runner: watch pass failed (exit=${exitCode}); staying alive, retrying next poll\n`);
3115
3395
  }
3116
3396
  const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
3117
3397
  ? exitCode === 0
@@ -3215,6 +3495,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3215
3495
  releaseWatcherJournalStores();
3216
3496
  releaseCachedAreaJournalMigrations();
3217
3497
  await shutdown();
3498
+ // Coordinator shutdown waits for any active pass before the retained
3499
+ // projections are dropped. A shutdown must never evict a store that a
3500
+ // final scoped writer is still refreshing or committing through.
3501
+ disposeScopedJournalStores();
3218
3502
  detachSignal?.();
3219
3503
  }
3220
3504
  }