@indigoai-us/hq-cloud 6.16.53 → 6.16.55-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +1 -23
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +8 -92
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +0 -36
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +1 -2
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +57 -126
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +403 -164
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +14 -4
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +59 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1553 -192
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +0 -40
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +162 -1454
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-rsync-preflight.test.js +4 -256
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +0 -4
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +1 -20
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +1 -16
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +0 -2
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +150 -467
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +95 -608
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +13 -15
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +226 -118
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +215 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +0 -2
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +0 -30
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +0 -175
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +259 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +43 -15
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +297 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +0 -7
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/lib/describe-error.test.js +0 -11
- package/dist/lib/describe-error.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +0 -295
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +3 -18
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +24 -47
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/s3.test.js +1 -27
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +2 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +66 -34
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +90 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync-core.d.ts +4 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +4 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +1 -25
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -26
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +20 -43
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +0 -32
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +12 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +131 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +63 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/cli/rescue-breaker.test.d.ts +0 -2
- package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
- package/dist/cli/rescue-breaker.test.js +0 -334
- package/dist/cli/rescue-breaker.test.js.map +0 -1
- package/dist/cli/rescue-realpath.test.d.ts +0 -2
- package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
- package/dist/cli/rescue-realpath.test.js +0 -109
- package/dist/cli/rescue-realpath.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.js +0 -627
- package/dist/cli/rescue-snapshot-copy.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
- package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.js +0 -210
- package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
|
@@ -10,11 +10,11 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
|
10
10
|
import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
|
-
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
13
|
+
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
14
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice,
|
|
17
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
18
18
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
19
19
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
20
20
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
@@ -30,6 +30,219 @@ import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
|
30
30
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
31
31
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
32
32
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
33
|
+
/** A deferred full chore must get the next admission after this long. */
|
|
34
|
+
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
35
|
+
/**
|
|
36
|
+
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
37
|
+
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
38
|
+
* durable unit, then resumes once the hand-off is finished. A separately
|
|
39
|
+
* tracked, explicitly opted-in watcher fast pass may use the pre-existing
|
|
40
|
+
* scoped fast-over-slow exception, but never while a chore owns durable state.
|
|
41
|
+
*/
|
|
42
|
+
export class CooperativePassScheduler {
|
|
43
|
+
now;
|
|
44
|
+
starvationCapMs;
|
|
45
|
+
pending = [];
|
|
46
|
+
active = null;
|
|
47
|
+
// Scoped fast realtime work has the established in-process lock exception
|
|
48
|
+
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
49
|
+
// from `active`: a chore still has exactly one durable-state owner.
|
|
50
|
+
activeFastRealtime = null;
|
|
51
|
+
stopped = false;
|
|
52
|
+
_handoffActive = false;
|
|
53
|
+
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
54
|
+
this.now = now;
|
|
55
|
+
this.starvationCapMs = starvationCapMs;
|
|
56
|
+
}
|
|
57
|
+
get handoffActive() {
|
|
58
|
+
return this._handoffActive;
|
|
59
|
+
}
|
|
60
|
+
get hasActiveChore() {
|
|
61
|
+
return this.active?.workClass === "chore";
|
|
62
|
+
}
|
|
63
|
+
get hasActiveSlowRealtime() {
|
|
64
|
+
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
65
|
+
}
|
|
66
|
+
pendingLaneCount(lane) {
|
|
67
|
+
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
68
|
+
}
|
|
69
|
+
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
70
|
+
if (this.stopped)
|
|
71
|
+
return Promise.resolve(0);
|
|
72
|
+
const directEntry = {
|
|
73
|
+
task,
|
|
74
|
+
workClass,
|
|
75
|
+
lane,
|
|
76
|
+
allowFastRealtimeOvertake,
|
|
77
|
+
// An immediately admitted task cannot starve, so avoid sampling the
|
|
78
|
+
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
79
|
+
// accounting and keeps the timestamp meaningful only for real queues.
|
|
80
|
+
enqueuedAt: 0,
|
|
81
|
+
resolve: () => undefined,
|
|
82
|
+
reject: () => undefined,
|
|
83
|
+
};
|
|
84
|
+
if (this.canStartFastRealtime(directEntry)) {
|
|
85
|
+
return this.startFastRealtime(directEntry);
|
|
86
|
+
}
|
|
87
|
+
if (this.active === null &&
|
|
88
|
+
this.activeFastRealtime === null &&
|
|
89
|
+
this.pending.length === 0) {
|
|
90
|
+
return this.start(directEntry);
|
|
91
|
+
}
|
|
92
|
+
return new Promise((resolve, reject) => {
|
|
93
|
+
const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
|
|
94
|
+
this.pending.push(entry);
|
|
95
|
+
this.drain();
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
stop() {
|
|
99
|
+
this.stopped = true;
|
|
100
|
+
while (this.pending.length > 0)
|
|
101
|
+
this.pending.shift()?.resolve(0);
|
|
102
|
+
}
|
|
103
|
+
drain() {
|
|
104
|
+
if (this.stopped || this.pending.length === 0)
|
|
105
|
+
return;
|
|
106
|
+
if (this.active !== null) {
|
|
107
|
+
const fastRealtime = this.nextFastRealtime();
|
|
108
|
+
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
109
|
+
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
110
|
+
this.startFastRealtime(fastRealtime);
|
|
111
|
+
}
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (this.activeFastRealtime !== null)
|
|
115
|
+
return;
|
|
116
|
+
const next = this.takeNext();
|
|
117
|
+
if (next)
|
|
118
|
+
this.start(next);
|
|
119
|
+
}
|
|
120
|
+
takeNext() {
|
|
121
|
+
const now = this.now();
|
|
122
|
+
const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
123
|
+
if (starvedChore >= 0)
|
|
124
|
+
return this.pending.splice(starvedChore, 1)[0];
|
|
125
|
+
// A receiver's bulk drain is still realtime work, but its slow lane is the
|
|
126
|
+
// progress boundary for that delivery. Preserve its established turn
|
|
127
|
+
// before refilling later fast realtime arrivals; this does not let a
|
|
128
|
+
// chore overtake any realtime work.
|
|
129
|
+
const realtime = this.nextRealtimeIndex();
|
|
130
|
+
if (realtime >= 0)
|
|
131
|
+
return this.pending.splice(realtime, 1)[0];
|
|
132
|
+
return this.pending.shift();
|
|
133
|
+
}
|
|
134
|
+
nextRealtimeIndex() {
|
|
135
|
+
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
136
|
+
return slowRealtime >= 0
|
|
137
|
+
? slowRealtime
|
|
138
|
+
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
139
|
+
}
|
|
140
|
+
nextFastRealtime() {
|
|
141
|
+
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
142
|
+
}
|
|
143
|
+
hasStarvedChore() {
|
|
144
|
+
const now = this.now();
|
|
145
|
+
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
146
|
+
}
|
|
147
|
+
canStartFastRealtime(entry) {
|
|
148
|
+
return (!this._handoffActive &&
|
|
149
|
+
this.activeFastRealtime === null &&
|
|
150
|
+
entry.workClass === "realtime" &&
|
|
151
|
+
entry.lane === "fast" &&
|
|
152
|
+
entry.allowFastRealtimeOvertake &&
|
|
153
|
+
this.hasActiveSlowRealtime &&
|
|
154
|
+
!this.hasStarvedChore());
|
|
155
|
+
}
|
|
156
|
+
start(entry) {
|
|
157
|
+
this.active = entry;
|
|
158
|
+
let current;
|
|
159
|
+
try {
|
|
160
|
+
current = this.execute(entry);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
current = Promise.reject(error);
|
|
164
|
+
}
|
|
165
|
+
void current.then((outcome) => {
|
|
166
|
+
entry.resolve(outcome);
|
|
167
|
+
this.complete(entry);
|
|
168
|
+
}, (error) => {
|
|
169
|
+
entry.reject(error);
|
|
170
|
+
this.complete(entry);
|
|
171
|
+
}).catch(() => undefined);
|
|
172
|
+
return current;
|
|
173
|
+
}
|
|
174
|
+
complete(entry) {
|
|
175
|
+
if (this.active === entry)
|
|
176
|
+
this.active = null;
|
|
177
|
+
this.drain();
|
|
178
|
+
}
|
|
179
|
+
startFastRealtime(entry) {
|
|
180
|
+
this.activeFastRealtime = entry;
|
|
181
|
+
let current;
|
|
182
|
+
try {
|
|
183
|
+
current = this.execute(entry);
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
current = Promise.reject(error);
|
|
187
|
+
}
|
|
188
|
+
void current.then((outcome) => {
|
|
189
|
+
entry.resolve(outcome);
|
|
190
|
+
this.completeFastRealtime(entry);
|
|
191
|
+
}, (error) => {
|
|
192
|
+
entry.reject(error);
|
|
193
|
+
this.completeFastRealtime(entry);
|
|
194
|
+
});
|
|
195
|
+
return current;
|
|
196
|
+
}
|
|
197
|
+
completeFastRealtime(entry) {
|
|
198
|
+
if (this.activeFastRealtime === entry)
|
|
199
|
+
this.activeFastRealtime = null;
|
|
200
|
+
this.drain();
|
|
201
|
+
}
|
|
202
|
+
execute(entry) {
|
|
203
|
+
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
204
|
+
}
|
|
205
|
+
async checkpoint(entry) {
|
|
206
|
+
if (this.stopped ||
|
|
207
|
+
entry.workClass !== "chore" ||
|
|
208
|
+
this.active !== entry ||
|
|
209
|
+
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
213
|
+
// the sole full-pass continuation and will resume after this bounded burst.
|
|
214
|
+
const deferredAt = this.now();
|
|
215
|
+
this.active = null;
|
|
216
|
+
this._handoffActive = true;
|
|
217
|
+
let handedOff = false;
|
|
218
|
+
try {
|
|
219
|
+
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
220
|
+
const realtimeIndex = this.nextRealtimeIndex();
|
|
221
|
+
if (realtimeIndex < 0)
|
|
222
|
+
break;
|
|
223
|
+
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
224
|
+
handedOff = true;
|
|
225
|
+
this.active = realtime;
|
|
226
|
+
try {
|
|
227
|
+
realtime.resolve(await this.execute(realtime));
|
|
228
|
+
}
|
|
229
|
+
catch (error) {
|
|
230
|
+
realtime.reject(error);
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
if (this.active === realtime)
|
|
234
|
+
this.active = null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
finally {
|
|
239
|
+
this._handoffActive = false;
|
|
240
|
+
if (!this.stopped)
|
|
241
|
+
this.active = entry;
|
|
242
|
+
}
|
|
243
|
+
return handedOff;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
33
246
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
34
247
|
/**
|
|
35
248
|
* Area storage is AUTO by default and only applies after durable cutover.
|
|
@@ -509,6 +722,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
509
722
|
const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
|
|
510
723
|
const monotonicNow = deps.monotonicNow ?? (() => performance.now());
|
|
511
724
|
const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
725
|
+
const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
|
|
512
726
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
513
727
|
// An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
|
|
514
728
|
// one). When it is omitted, the interval is resolved fresh each cycle from
|
|
@@ -672,7 +886,27 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
672
886
|
resolveNoOperationLockUsers = resolve;
|
|
673
887
|
});
|
|
674
888
|
};
|
|
675
|
-
const
|
|
889
|
+
const passScheduler = new CooperativePassScheduler(monotonicNow);
|
|
890
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
891
|
+
const reportedAppliedUpserts = new Set();
|
|
892
|
+
const recordDurableAppliedUpserts = (upserts) => {
|
|
893
|
+
for (const applied of upserts) {
|
|
894
|
+
const key = `${applied.relativePath}\u0000${applied.contentHash}`;
|
|
895
|
+
if (reportedAppliedUpserts.has(key))
|
|
896
|
+
continue;
|
|
897
|
+
reportedAppliedUpserts.add(key);
|
|
898
|
+
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
899
|
+
}
|
|
900
|
+
};
|
|
901
|
+
const invokePass = () => {
|
|
902
|
+
if (!runPassSupportsCooperativeCheckpoint)
|
|
903
|
+
return runPass(passArgvForRun);
|
|
904
|
+
// This callback is emitted by the production runner after journal
|
|
905
|
+
// repair/seed and before fanout. It starts ingress early, but never
|
|
906
|
+
// admits a second durable writer: queued realtime still waits for the
|
|
907
|
+
// existing scheduler's cooperative hand-off checkpoint.
|
|
908
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
909
|
+
};
|
|
676
910
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
677
911
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
678
912
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -684,9 +918,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
684
918
|
heapCheckpoint?.();
|
|
685
919
|
if (typeof outcome === "number")
|
|
686
920
|
return;
|
|
687
|
-
|
|
688
|
-
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
689
|
-
}
|
|
921
|
+
recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
|
|
690
922
|
};
|
|
691
923
|
// Pre-admit boundary observation.
|
|
692
924
|
heapCheckpoint?.();
|
|
@@ -695,13 +927,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
695
927
|
// its keyed journal delta and per-object mutation are independent from the
|
|
696
928
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
697
929
|
// Never bypass the lock when no slow pass is active.
|
|
930
|
+
if (
|
|
931
|
+
// A full chore is paused at a caller-declared durable checkpoint. Keep
|
|
932
|
+
// the root lock held by this process, but lend it to exactly one
|
|
933
|
+
// realtime pass; the suspended chore cannot mutate until this returns.
|
|
934
|
+
workClass === "realtime" &&
|
|
935
|
+
passScheduler.handoffActive &&
|
|
936
|
+
processOwnsOperationLock) {
|
|
937
|
+
acquireOperationLockUser();
|
|
938
|
+
try {
|
|
939
|
+
prepare?.();
|
|
940
|
+
const outcome = await invokePass();
|
|
941
|
+
recordAppliedUpserts(outcome);
|
|
942
|
+
return outcome;
|
|
943
|
+
}
|
|
944
|
+
finally {
|
|
945
|
+
releaseOperationLockUser();
|
|
946
|
+
}
|
|
947
|
+
}
|
|
698
948
|
if (lane === "fast" &&
|
|
699
949
|
processOwnsOperationLock &&
|
|
700
950
|
coordinatorWholeMutationChores === 0) {
|
|
701
951
|
acquireOperationLockUser();
|
|
702
952
|
try {
|
|
703
953
|
prepare?.();
|
|
704
|
-
const outcome = await
|
|
954
|
+
const outcome = await invokePass();
|
|
705
955
|
recordAppliedUpserts(outcome);
|
|
706
956
|
return outcome;
|
|
707
957
|
}
|
|
@@ -721,7 +971,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
721
971
|
acquireOperationLockUser();
|
|
722
972
|
}
|
|
723
973
|
prepare?.();
|
|
724
|
-
const outcome = await
|
|
974
|
+
const outcome = await invokePass();
|
|
725
975
|
recordAppliedUpserts(outcome);
|
|
726
976
|
return outcome;
|
|
727
977
|
}
|
|
@@ -755,7 +1005,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
755
1005
|
}
|
|
756
1006
|
try {
|
|
757
1007
|
prepare?.();
|
|
758
|
-
const outcome = await
|
|
1008
|
+
const outcome = await invokePass();
|
|
759
1009
|
recordAppliedUpserts(outcome);
|
|
760
1010
|
return outcome;
|
|
761
1011
|
}
|
|
@@ -778,7 +1028,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
778
1028
|
throw err;
|
|
779
1029
|
}
|
|
780
1030
|
};
|
|
781
|
-
const runTaskWithLock = async (task) => {
|
|
1031
|
+
const runTaskWithLock = async (task, workClass = "chore") => {
|
|
1032
|
+
if (workClass === "realtime" &&
|
|
1033
|
+
passScheduler.handoffActive &&
|
|
1034
|
+
processOwnsOperationLock) {
|
|
1035
|
+
acquireOperationLockUser();
|
|
1036
|
+
try {
|
|
1037
|
+
return await task();
|
|
1038
|
+
}
|
|
1039
|
+
finally {
|
|
1040
|
+
releaseOperationLockUser();
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
782
1043
|
try {
|
|
783
1044
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
784
1045
|
timeoutSec: 0,
|
|
@@ -872,102 +1133,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
872
1133
|
heapCheckpoint?.();
|
|
873
1134
|
}
|
|
874
1135
|
};
|
|
875
|
-
let activeSlowPass = null;
|
|
876
|
-
let activeFastPass = null;
|
|
877
|
-
const pendingSlowPasses = [];
|
|
878
|
-
const pendingFastPasses = [];
|
|
879
1136
|
const resolveStoppedQueue = () => {
|
|
880
|
-
|
|
881
|
-
while (queue.length > 0)
|
|
882
|
-
queue.shift()?.resolve(0);
|
|
883
|
-
}
|
|
1137
|
+
passScheduler.stop();
|
|
884
1138
|
};
|
|
885
|
-
const
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
// refilling the realtime queue; otherwise a steady stream of fast work
|
|
892
|
-
// perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
|
|
893
|
-
// cannot issue another receive.
|
|
894
|
-
if (activeSlowPass === null &&
|
|
895
|
-
activeFastPass === null &&
|
|
896
|
-
pendingSlowPasses.length > 0) {
|
|
897
|
-
const slow = pendingSlowPasses.shift();
|
|
898
|
-
if (slow) {
|
|
899
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
900
|
-
void current.then(slow.resolve, slow.reject);
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
if (activeFastPass === null) {
|
|
904
|
-
const fast = pendingFastPasses.shift();
|
|
905
|
-
if (fast) {
|
|
906
|
-
const current = startGuardedTask(fast.task, "fast");
|
|
907
|
-
void current.then(fast.resolve, fast.reject);
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
// A fast pass that acquired the operation lock normally (rather than
|
|
911
|
-
// bypassing an in-process slow holder) still owns the cross-process
|
|
912
|
-
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
913
|
-
// overlap an already-lock-owning slow pass.
|
|
914
|
-
if (activeSlowPass === null && activeFastPass === null) {
|
|
915
|
-
const slow = pendingSlowPasses.shift();
|
|
916
|
-
if (slow) {
|
|
917
|
-
const current = startGuardedTask(slow.task, "slow");
|
|
918
|
-
void current.then(slow.resolve, slow.reject);
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
};
|
|
922
|
-
const startGuardedTask = (task, lane) => {
|
|
923
|
-
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
924
|
-
// a watcher/scheduler callback before returning its promise. Without this
|
|
925
|
-
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
926
|
-
// second lock contender.
|
|
927
|
-
const placeholder = Promise.resolve(0);
|
|
928
|
-
if (lane === "fast")
|
|
929
|
-
activeFastPass = placeholder;
|
|
930
|
-
else
|
|
931
|
-
activeSlowPass = placeholder;
|
|
932
|
-
let current;
|
|
933
|
-
try {
|
|
934
|
-
current = stopped ? placeholder : task();
|
|
935
|
-
}
|
|
936
|
-
catch (err) {
|
|
937
|
-
current = Promise.reject(err);
|
|
938
|
-
}
|
|
939
|
-
if (lane === "fast")
|
|
940
|
-
activeFastPass = current;
|
|
941
|
-
else
|
|
942
|
-
activeSlowPass = current;
|
|
943
|
-
void current
|
|
944
|
-
.finally(() => {
|
|
945
|
-
if (lane === "fast" && activeFastPass === current) {
|
|
946
|
-
activeFastPass = null;
|
|
947
|
-
drainQueuedPasses();
|
|
948
|
-
}
|
|
949
|
-
else if (lane === "slow" && activeSlowPass === current) {
|
|
950
|
-
activeSlowPass = null;
|
|
951
|
-
drainQueuedPasses();
|
|
952
|
-
}
|
|
953
|
-
})
|
|
954
|
-
.catch(() => undefined);
|
|
955
|
-
return current;
|
|
956
|
-
};
|
|
957
|
-
const runGuardedTask = (task, lane = "slow") => {
|
|
958
|
-
const active = lane === "fast"
|
|
959
|
-
? activeFastPass
|
|
960
|
-
: activeSlowPass ?? activeFastPass;
|
|
961
|
-
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
962
|
-
if (active === null && queued.length === 0) {
|
|
963
|
-
return startGuardedTask(task, lane);
|
|
964
|
-
}
|
|
965
|
-
return new Promise((resolve, reject) => {
|
|
966
|
-
queued.push({ task, lane, resolve, reject });
|
|
967
|
-
drainQueuedPasses();
|
|
968
|
-
});
|
|
969
|
-
};
|
|
970
|
-
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
1139
|
+
const runGuardedTask = (task, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) =>
|
|
1140
|
+
// Lanes still choose the pass execution profile, but no longer decide
|
|
1141
|
+
// chore ownership. Realtime normally waits for a chore hand-off; the
|
|
1142
|
+
// watcher fast path explicitly opts into its established scoped exception.
|
|
1143
|
+
passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake);
|
|
1144
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false) => runGuardedTask((context) => runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined), lane, workClass, allowFastRealtimeOvertake);
|
|
971
1145
|
/*
|
|
972
1146
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
973
1147
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1159,6 +1333,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1159
1333
|
});
|
|
1160
1334
|
},
|
|
1161
1335
|
onCutOver: () => {
|
|
1336
|
+
releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1162
1337
|
parallelCoordinatorEnabled = refreshCoordinatorEnabled();
|
|
1163
1338
|
if (parallelCoordinatorEnabled && coordinator === null) {
|
|
1164
1339
|
coordinator = createCoordinator();
|
|
@@ -1245,6 +1420,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1245
1420
|
let drainedUntrackedDeletes = 0;
|
|
1246
1421
|
const pendingWatcherPaths = new Map();
|
|
1247
1422
|
const pendingWatcherChanges = new Map();
|
|
1423
|
+
// A warm-up scan observes filesystem state over time. Keep an ingress
|
|
1424
|
+
// sequence so its older observations cannot overwrite a live watcher event
|
|
1425
|
+
// that arrived while the scan was yielding.
|
|
1426
|
+
const pendingWatcherChangeSequences = new Map();
|
|
1427
|
+
let nextPendingWatcherChangeSequence = 0;
|
|
1248
1428
|
let pendingWatcherOriginalBatch = null;
|
|
1249
1429
|
// Every coalesced watcher batch gets one durable in-process generation.
|
|
1250
1430
|
// A wake can cause the scheduler to revisit a drained batch, but an event
|
|
@@ -1268,11 +1448,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1268
1448
|
const paths = new Map();
|
|
1269
1449
|
const changes = new Map();
|
|
1270
1450
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1271
|
-
const
|
|
1451
|
+
const change = batch.changes?.get(absolutePath);
|
|
1452
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1272
1453
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1273
1454
|
continue;
|
|
1274
1455
|
paths.set(absolutePath, relativePath);
|
|
1275
|
-
const change = batch.changes?.get(absolutePath);
|
|
1276
1456
|
if (change)
|
|
1277
1457
|
changes.set(absolutePath, change);
|
|
1278
1458
|
}
|
|
@@ -1280,7 +1460,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1280
1460
|
return null;
|
|
1281
1461
|
return { ...batch, paths, changes };
|
|
1282
1462
|
};
|
|
1283
|
-
const addPendingWatcherChange = (changedRelPath, batch) => {
|
|
1463
|
+
const addPendingWatcherChange = (changedRelPath, batch, options) => {
|
|
1284
1464
|
if (batch) {
|
|
1285
1465
|
const selectedBatch = selectWatchBatch(batch);
|
|
1286
1466
|
if (!selectedBatch)
|
|
@@ -1293,20 +1473,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1293
1473
|
return false;
|
|
1294
1474
|
}
|
|
1295
1475
|
}
|
|
1296
|
-
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1297
1476
|
if (batch) {
|
|
1477
|
+
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
1478
|
+
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
1479
|
+
return previousSequence === undefined ||
|
|
1480
|
+
options?.preserveChangesAfter === undefined ||
|
|
1481
|
+
previousSequence <= options.preserveChangesAfter;
|
|
1482
|
+
});
|
|
1483
|
+
if (entries.length === 0)
|
|
1484
|
+
return false;
|
|
1485
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1298
1486
|
pendingWatcherOriginalBatch =
|
|
1299
1487
|
pendingWatcherPaths.size === 0 &&
|
|
1300
1488
|
!pendingWatcherBareChange &&
|
|
1301
1489
|
!pendingWatcherOverflowed &&
|
|
1302
|
-
!batch.overflowed
|
|
1490
|
+
!batch.overflowed &&
|
|
1491
|
+
entries.length === batch.paths.size
|
|
1303
1492
|
? batch
|
|
1304
1493
|
: null;
|
|
1305
|
-
for (const [absolutePath, relativePath] of
|
|
1494
|
+
for (const [absolutePath, relativePath] of entries) {
|
|
1306
1495
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1307
1496
|
const change = batch.changes?.get(absolutePath);
|
|
1308
1497
|
if (change)
|
|
1309
1498
|
pendingWatcherChanges.set(absolutePath, change);
|
|
1499
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1310
1500
|
}
|
|
1311
1501
|
if (batch.overflowed) {
|
|
1312
1502
|
pendingWatcherOverflowed = true;
|
|
@@ -1323,6 +1513,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1323
1513
|
}
|
|
1324
1514
|
return true;
|
|
1325
1515
|
}
|
|
1516
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1326
1517
|
if (changedRelPath) {
|
|
1327
1518
|
pendingWatcherOriginalBatch = null;
|
|
1328
1519
|
// Codec boundary: `changedRelPath` is a canonical vault key (the
|
|
@@ -1331,6 +1522,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1331
1522
|
// point at the impossible colon path on Windows.
|
|
1332
1523
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
1333
1524
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
1525
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1334
1526
|
// A bare path signal carries no delete intent; drop any change record
|
|
1335
1527
|
// a prior batch left for this path so the stale intent cannot leak
|
|
1336
1528
|
// into the coalesced batch (#227).
|
|
@@ -1372,6 +1564,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1372
1564
|
const generation = pendingWatcherBatchGeneration;
|
|
1373
1565
|
pendingWatcherPaths.clear();
|
|
1374
1566
|
pendingWatcherChanges.clear();
|
|
1567
|
+
pendingWatcherChangeSequences.clear();
|
|
1375
1568
|
pendingWatcherOriginalBatch = null;
|
|
1376
1569
|
pendingWatcherBareChange = false;
|
|
1377
1570
|
pendingWatcherWarmupCatchup = false;
|
|
@@ -1390,6 +1583,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1390
1583
|
pendingWatcherOriginalBatch = null;
|
|
1391
1584
|
for (const [absolutePath, relativePath] of paths.entries()) {
|
|
1392
1585
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1586
|
+
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1393
1587
|
}
|
|
1394
1588
|
};
|
|
1395
1589
|
/**
|
|
@@ -1470,8 +1664,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1470
1664
|
}
|
|
1471
1665
|
return null;
|
|
1472
1666
|
};
|
|
1473
|
-
const journalCoordinates = (relativePath) => {
|
|
1474
|
-
const route = routeChangeToTarget(relativePath);
|
|
1667
|
+
const journalCoordinates = (relativePath, kind) => {
|
|
1668
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1475
1669
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1476
1670
|
return null;
|
|
1477
1671
|
if (route.kind === "personal") {
|
|
@@ -1506,7 +1700,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1506
1700
|
declaredSize >= 0) {
|
|
1507
1701
|
return declaredSize;
|
|
1508
1702
|
}
|
|
1509
|
-
const size = journalRows(
|
|
1703
|
+
const size = journalRows(readJournalScoped(coordinates.slug, [coordinates.key]))
|
|
1704
|
+
.get(coordinates.key)?.size;
|
|
1510
1705
|
return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
|
|
1511
1706
|
};
|
|
1512
1707
|
/**
|
|
@@ -1553,8 +1748,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1553
1748
|
return session.get(coordinates.key) === undefined;
|
|
1554
1749
|
};
|
|
1555
1750
|
const captureLocalDeleteSnapshots = (relativePath, kind) => {
|
|
1556
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1557
|
-
const route = routeChangeToTarget(relativePath);
|
|
1751
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1752
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1558
1753
|
if (!coordinates || !route)
|
|
1559
1754
|
return [];
|
|
1560
1755
|
// Desktop workspace Off toggles are authoritative before journal I/O.
|
|
@@ -1675,8 +1870,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1675
1870
|
* it scopes a push or pull for the affected batch.
|
|
1676
1871
|
*/
|
|
1677
1872
|
const captureQueuedLocalDeleteSnapshots = async (relativePath, kind, isCurrent) => {
|
|
1678
|
-
const coordinates = journalCoordinates(relativePath);
|
|
1679
|
-
const route = routeChangeToTarget(relativePath);
|
|
1873
|
+
const coordinates = journalCoordinates(relativePath, kind);
|
|
1874
|
+
const route = routeChangeToTarget(relativePath, kind);
|
|
1680
1875
|
if (!coordinates || !route)
|
|
1681
1876
|
return [];
|
|
1682
1877
|
if (route.kind === "company" &&
|
|
@@ -1770,8 +1965,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1770
1965
|
}
|
|
1771
1966
|
return snapshots;
|
|
1772
1967
|
};
|
|
1773
|
-
const warmupJournalPaths = ()
|
|
1774
|
-
const paths = [];
|
|
1968
|
+
const warmupJournalPaths = function* () {
|
|
1775
1969
|
const journals = listJournals();
|
|
1776
1970
|
const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1777
1971
|
for (const { slug, journal } of journals) {
|
|
@@ -1804,10 +1998,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1804
1998
|
// ~12.8k of them per runner start after a transcript prune.
|
|
1805
1999
|
if (route.kind === "personal" && isPersonalVaultDeletionExcluded(key))
|
|
1806
2000
|
continue;
|
|
1807
|
-
|
|
2001
|
+
yield route.kind === "personal" ? key : `companies/${route.slug}/${key}`;
|
|
1808
2002
|
}
|
|
1809
2003
|
}
|
|
1810
|
-
return paths;
|
|
1811
2004
|
};
|
|
1812
2005
|
const publishedContentHashLookup = (relativePath) => {
|
|
1813
2006
|
const coordinates = journalCoordinates(relativePath);
|
|
@@ -1820,13 +2013,34 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1820
2013
|
return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
|
|
1821
2014
|
};
|
|
1822
2015
|
const prepareWatcherChanges = (changes) => {
|
|
2016
|
+
const keysByJournal = new Map();
|
|
2017
|
+
const rememberKey = (slug, key) => {
|
|
2018
|
+
const keys = keysByJournal.get(slug) ?? new Set();
|
|
2019
|
+
keys.add(key);
|
|
2020
|
+
keysByJournal.set(slug, keys);
|
|
2021
|
+
};
|
|
2022
|
+
for (const [absolutePath, change] of changes.entries()) {
|
|
2023
|
+
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
2024
|
+
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
2025
|
+
rememberKey(snapshot.journalSlug, snapshot.journalPath);
|
|
2026
|
+
}
|
|
2027
|
+
continue;
|
|
2028
|
+
}
|
|
2029
|
+
const relativePath = path
|
|
2030
|
+
.relative(hqRoot, absolutePath)
|
|
2031
|
+
.split(path.sep)
|
|
2032
|
+
.join("/");
|
|
2033
|
+
const coordinates = journalCoordinates(relativePath);
|
|
2034
|
+
if (coordinates)
|
|
2035
|
+
rememberKey(coordinates.slug, coordinates.key);
|
|
2036
|
+
}
|
|
1823
2037
|
const journals = new Map();
|
|
1824
2038
|
const dirty = new Set();
|
|
1825
2039
|
const journalFor = (slug) => {
|
|
1826
2040
|
const loaded = journals.get(slug);
|
|
1827
2041
|
if (loaded)
|
|
1828
2042
|
return loaded;
|
|
1829
|
-
const journal =
|
|
2043
|
+
const journal = readJournalScoped(slug, [...(keysByJournal.get(slug) ?? [])]);
|
|
1830
2044
|
journals.set(slug, journal);
|
|
1831
2045
|
return journal;
|
|
1832
2046
|
};
|
|
@@ -1884,7 +2098,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1884
2098
|
const groupBatchByRoute = (batch) => {
|
|
1885
2099
|
const grouped = new Map();
|
|
1886
2100
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1887
|
-
const
|
|
2101
|
+
const change = batch.changes?.get(absolutePath);
|
|
2102
|
+
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1888
2103
|
if (!route)
|
|
1889
2104
|
continue;
|
|
1890
2105
|
const key = routeKey(route);
|
|
@@ -1897,7 +2112,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1897
2112
|
deleteScopeRoots: [],
|
|
1898
2113
|
};
|
|
1899
2114
|
existing.paths.set(absolutePath, relativePath);
|
|
1900
|
-
const change = batch.changes?.get(absolutePath);
|
|
1901
2115
|
if (change)
|
|
1902
2116
|
existing.changes.set(absolutePath, change);
|
|
1903
2117
|
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
@@ -1981,7 +2195,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1981
2195
|
}
|
|
1982
2196
|
return false;
|
|
1983
2197
|
};
|
|
1984
|
-
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
2198
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
|
|
1985
2199
|
if (batch)
|
|
1986
2200
|
await resolveDeleteSnapshotCaptures(batch);
|
|
1987
2201
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
@@ -2010,11 +2224,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2010
2224
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2011
2225
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2012
2226
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2013
|
-
: undefined, lane);
|
|
2227
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake);
|
|
2014
2228
|
})
|
|
2015
2229
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2016
2230
|
? () => prepareWatcherChanges(group.changes)
|
|
2017
|
-
: undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
|
|
2231
|
+
: undefined, lane, "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
|
|
2018
2232
|
const result = execution.result;
|
|
2019
2233
|
if (execution.accepted && passExitCode(result) === 0) {
|
|
2020
2234
|
const refusedPaths = new Set(typeof result === "number"
|
|
@@ -2084,8 +2298,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2084
2298
|
? "fast"
|
|
2085
2299
|
: "slow";
|
|
2086
2300
|
const execution = parallelCoordinatorEnabled
|
|
2087
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
|
|
2088
|
-
: { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
|
|
2301
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))
|
|
2302
|
+
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
2089
2303
|
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
2090
2304
|
if (exit !== 0 && worstExit === 0)
|
|
2091
2305
|
worstExit = exit;
|
|
@@ -2290,13 +2504,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2290
2504
|
return;
|
|
2291
2505
|
}
|
|
2292
2506
|
if (lanes?.fast &&
|
|
2293
|
-
|
|
2507
|
+
(passScheduler.hasActiveChore || passScheduler.hasActiveSlowRealtime)) {
|
|
2294
2508
|
const generation = ++nextWatcherBatchGeneration;
|
|
2295
2509
|
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
2296
2510
|
// of its contracts here: only selected routes may run, and V2 sees
|
|
2297
2511
|
// the same dirty signal as the compatibility drain.
|
|
2298
2512
|
realtimeState.scheduler?.signal("watcher");
|
|
2299
|
-
void runScopedDrain(lanes.fast, generation, true).then((exit) => {
|
|
2513
|
+
void runScopedDrain(lanes.fast, generation, true, true).then((exit) => {
|
|
2300
2514
|
if (exit === 0)
|
|
2301
2515
|
return;
|
|
2302
2516
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
@@ -2345,29 +2559,36 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2345
2559
|
watcherRediscoveryRetries = 0;
|
|
2346
2560
|
process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
|
|
2347
2561
|
const startedAt = Date.now();
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2562
|
+
const liveChangeSequenceAtWarmupStart = nextPendingWatcherChangeSequence;
|
|
2563
|
+
void (async () => {
|
|
2564
|
+
try {
|
|
2565
|
+
const catchup = watcher?.collectWarmupCatchupAsync
|
|
2566
|
+
? await watcher.collectWarmupCatchupAsync(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths())
|
|
2567
|
+
: watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
|
|
2568
|
+
if (!catchup || catchup.cancelled || stopped || watcher === null)
|
|
2569
|
+
return;
|
|
2570
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch, {
|
|
2571
|
+
preserveChangesAfter: liveChangeSequenceAtWarmupStart,
|
|
2572
|
+
});
|
|
2573
|
+
pendingWatcherWarmupCatchup ||= queued;
|
|
2574
|
+
process.stderr.write(JSON.stringify({
|
|
2575
|
+
type: "watcher-warmup-catchup",
|
|
2576
|
+
reason: "watch-set-ready",
|
|
2577
|
+
pathsFound: catchup.batch.paths.size,
|
|
2578
|
+
scannedPaths: catchup.scannedPaths,
|
|
2579
|
+
elapsedMs: Date.now() - startedAt,
|
|
2580
|
+
queued,
|
|
2581
|
+
}) + "\n");
|
|
2582
|
+
if (queued)
|
|
2583
|
+
interruptWait();
|
|
2584
|
+
}
|
|
2585
|
+
catch (err) {
|
|
2586
|
+
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2587
|
+
// A scan failure must not silently extend the warm-up blind spot.
|
|
2588
|
+
pendingWatcherWarmupCatchupFailed = true;
|
|
2363
2589
|
interruptWait();
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2367
|
-
// A scan failure must not silently extend the warm-up blind spot.
|
|
2368
|
-
pendingWatcherWarmupCatchupFailed = true;
|
|
2369
|
-
interruptWait();
|
|
2370
|
-
}
|
|
2590
|
+
}
|
|
2591
|
+
})();
|
|
2371
2592
|
};
|
|
2372
2593
|
if (watcher.onReady)
|
|
2373
2594
|
watcher.onReady(reportWatcherReady);
|
|
@@ -2415,7 +2636,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2415
2636
|
// Submit every route in this lane before awaiting any result. In
|
|
2416
2637
|
// particular, a later route's fast receipt must never wait for an
|
|
2417
2638
|
// earlier route's slow pull to finish.
|
|
2418
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
|
|
2639
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime"))));
|
|
2419
2640
|
for (const execution of completed) {
|
|
2420
2641
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2421
2642
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -2469,7 +2690,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2469
2690
|
}
|
|
2470
2691
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
2471
2692
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
2472
|
-
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
2693
|
+
const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
2473
2694
|
if (passExitCode(result) !== 0) {
|
|
2474
2695
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
2475
2696
|
}
|
|
@@ -2517,14 +2738,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2517
2738
|
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2518
2739
|
await drain();
|
|
2519
2740
|
return 0;
|
|
2520
|
-
})));
|
|
2741
|
+
}, "realtime"), "slow", "realtime"));
|
|
2521
2742
|
outcome = execution.accepted ? execution.result : 1;
|
|
2522
2743
|
}
|
|
2523
2744
|
else {
|
|
2524
2745
|
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2525
2746
|
await drain();
|
|
2526
2747
|
return 0;
|
|
2527
|
-
}));
|
|
2748
|
+
}, "realtime"), "slow", "realtime");
|
|
2528
2749
|
}
|
|
2529
2750
|
}
|
|
2530
2751
|
catch (err) {
|
|
@@ -2720,8 +2941,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2720
2941
|
targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
|
|
2721
2942
|
targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
|
|
2722
2943
|
cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
|
|
2723
|
-
pendingSlowPasses: () =>
|
|
2724
|
-
pendingFastPasses: () =>
|
|
2944
|
+
pendingSlowPasses: () => passScheduler.pendingLaneCount("slow"),
|
|
2945
|
+
pendingFastPasses: () => passScheduler.pendingLaneCount("fast"),
|
|
2725
2946
|
},
|
|
2726
2947
|
eventSyncSizes: () => eventSync?.censusSizes?.(),
|
|
2727
2948
|
watcherSizes: () => watcher?.censusSizes?.(),
|
|
@@ -3071,11 +3292,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3071
3292
|
quarantinedDeleteRoots.delete(root);
|
|
3072
3293
|
}
|
|
3073
3294
|
// Delete snapshot capture needs a fresh keyed view while an episode is
|
|
3074
|
-
// being assembled, but completed watch cycles do not.
|
|
3075
|
-
//
|
|
3076
|
-
//
|
|
3295
|
+
// being assembled, but completed watch cycles do not. Area-authoritative
|
|
3296
|
+
// journals are different: their bridge owns the packed aggregate that a
|
|
3297
|
+
// later watcher callback reuses. Dropping that bridge at every poll
|
|
3298
|
+
// makes the next scoped callback synchronously re-pack the whole vault.
|
|
3299
|
+
// Keep it for this runner's lifetime; layout/phase changes still close
|
|
3300
|
+
// it in openAreaMigrationIfAuthoritative(), and final shutdown releases
|
|
3301
|
+
// every bridge below.
|
|
3077
3302
|
releaseWatcherJournalStores();
|
|
3078
|
-
releaseCachedAreaJournalMigrations();
|
|
3079
3303
|
const exitCode = passExitCode(result);
|
|
3080
3304
|
if (exitCode === runtime.authRequiredPassExit) {
|
|
3081
3305
|
// The pass already emitted auth-error. Hand the sentinel to the
|
|
@@ -3110,8 +3334,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3110
3334
|
: "") +
|
|
3111
3335
|
"; staying alive, retrying next poll\n");
|
|
3112
3336
|
}
|
|
3337
|
+
else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
3338
|
+
// HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
|
|
3339
|
+
// company directory names, which may not be memberships. The runner
|
|
3340
|
+
// has already emitted the resolution error; terminating here would
|
|
3341
|
+
// restart the outpost watcher every 60 seconds for each such route.
|
|
3342
|
+
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
3343
|
+
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
3344
|
+
}
|
|
3113
3345
|
else if (exitCode !== 0) {
|
|
3114
|
-
|
|
3346
|
+
// A scoped company-resolution or per-pass failure is isolated to that
|
|
3347
|
+
// pass. The error was emitted by the runner; terminating the daemon
|
|
3348
|
+
// recreates the startup blind window and strands newer watcher work.
|
|
3349
|
+
process.stderr.write(`hq-sync-runner: watch pass failed (exit=${exitCode}); staying alive, retrying next poll\n`);
|
|
3115
3350
|
}
|
|
3116
3351
|
const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
|
|
3117
3352
|
? exitCode === 0
|
|
@@ -3215,6 +3450,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3215
3450
|
releaseWatcherJournalStores();
|
|
3216
3451
|
releaseCachedAreaJournalMigrations();
|
|
3217
3452
|
await shutdown();
|
|
3453
|
+
// Coordinator shutdown waits for any active pass before the retained
|
|
3454
|
+
// projections are dropped. A shutdown must never evict a store that a
|
|
3455
|
+
// final scoped writer is still refreshing or committing through.
|
|
3456
|
+
disposeScopedJournalStores();
|
|
3218
3457
|
detachSignal?.();
|
|
3219
3458
|
}
|
|
3220
3459
|
}
|