@indigoai-us/hq-cloud 6.16.55-alpha.0 → 6.16.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +23 -1
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +92 -8
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +36 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -14
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +29 -46
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +247 -57
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -51
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +164 -403
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +4 -14
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -59
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -48
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +221 -1520
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1454 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +627 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +467 -150
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +608 -95
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +15 -13
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/context.d.ts +5 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +92 -12
- package/dist/context.js.map +1 -1
- package/dist/context.test.js +77 -2
- package/dist/context.test.js.map +1 -1
- package/dist/ignore.d.ts +10 -0
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +47 -5
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +30 -3
- package/dist/ignore.test.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -40
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -240
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +0 -54
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -259
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +15 -43
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +79 -297
- package/dist/journal.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +7 -0
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/lib/describe-error.test.js +11 -0
- package/dist/lib/describe-error.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/lib/net-errors.d.ts +12 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +35 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +19 -1
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/manifest/build-manifest-resident-parity.test.js +3 -3
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +2 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +34 -66
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +5 -90
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +0 -6
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +2 -14
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +33 -5
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +69 -21
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +65 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +0 -12
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +0 -131
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +0 -63
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
|
@@ -10,11 +10,11 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
|
10
10
|
import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
|
-
import { PARTIAL_SYNC_EXIT
|
|
13
|
+
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
14
14
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice,
|
|
17
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, readJournal, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
18
18
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
19
19
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
20
20
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
@@ -30,219 +30,6 @@ import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
|
30
30
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
31
31
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
32
32
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
33
|
-
/** A deferred full chore must get the next admission after this long. */
|
|
34
|
-
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
35
|
-
/**
|
|
36
|
-
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
37
|
-
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
38
|
-
* durable unit, then resumes once the hand-off is finished. A separately
|
|
39
|
-
* tracked, explicitly opted-in watcher fast pass may use the pre-existing
|
|
40
|
-
* scoped fast-over-slow exception, but never while a chore owns durable state.
|
|
41
|
-
*/
|
|
42
|
-
export class CooperativePassScheduler {
|
|
43
|
-
now;
|
|
44
|
-
starvationCapMs;
|
|
45
|
-
pending = [];
|
|
46
|
-
active = null;
|
|
47
|
-
// Scoped fast realtime work has the established in-process lock exception
|
|
48
|
-
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
49
|
-
// from `active`: a chore still has exactly one durable-state owner.
|
|
50
|
-
activeFastRealtime = null;
|
|
51
|
-
stopped = false;
|
|
52
|
-
_handoffActive = false;
|
|
53
|
-
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
54
|
-
this.now = now;
|
|
55
|
-
this.starvationCapMs = starvationCapMs;
|
|
56
|
-
}
|
|
57
|
-
get handoffActive() {
|
|
58
|
-
return this._handoffActive;
|
|
59
|
-
}
|
|
60
|
-
get hasActiveChore() {
|
|
61
|
-
return this.active?.workClass === "chore";
|
|
62
|
-
}
|
|
63
|
-
get hasActiveSlowRealtime() {
|
|
64
|
-
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
65
|
-
}
|
|
66
|
-
pendingLaneCount(lane) {
|
|
67
|
-
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
68
|
-
}
|
|
69
|
-
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
70
|
-
if (this.stopped)
|
|
71
|
-
return Promise.resolve(0);
|
|
72
|
-
const directEntry = {
|
|
73
|
-
task,
|
|
74
|
-
workClass,
|
|
75
|
-
lane,
|
|
76
|
-
allowFastRealtimeOvertake,
|
|
77
|
-
// An immediately admitted task cannot starve, so avoid sampling the
|
|
78
|
-
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
79
|
-
// accounting and keeps the timestamp meaningful only for real queues.
|
|
80
|
-
enqueuedAt: 0,
|
|
81
|
-
resolve: () => undefined,
|
|
82
|
-
reject: () => undefined,
|
|
83
|
-
};
|
|
84
|
-
if (this.canStartFastRealtime(directEntry)) {
|
|
85
|
-
return this.startFastRealtime(directEntry);
|
|
86
|
-
}
|
|
87
|
-
if (this.active === null &&
|
|
88
|
-
this.activeFastRealtime === null &&
|
|
89
|
-
this.pending.length === 0) {
|
|
90
|
-
return this.start(directEntry);
|
|
91
|
-
}
|
|
92
|
-
return new Promise((resolve, reject) => {
|
|
93
|
-
const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
|
|
94
|
-
this.pending.push(entry);
|
|
95
|
-
this.drain();
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
stop() {
|
|
99
|
-
this.stopped = true;
|
|
100
|
-
while (this.pending.length > 0)
|
|
101
|
-
this.pending.shift()?.resolve(0);
|
|
102
|
-
}
|
|
103
|
-
drain() {
|
|
104
|
-
if (this.stopped || this.pending.length === 0)
|
|
105
|
-
return;
|
|
106
|
-
if (this.active !== null) {
|
|
107
|
-
const fastRealtime = this.nextFastRealtime();
|
|
108
|
-
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
109
|
-
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
110
|
-
this.startFastRealtime(fastRealtime);
|
|
111
|
-
}
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
if (this.activeFastRealtime !== null)
|
|
115
|
-
return;
|
|
116
|
-
const next = this.takeNext();
|
|
117
|
-
if (next)
|
|
118
|
-
this.start(next);
|
|
119
|
-
}
|
|
120
|
-
takeNext() {
|
|
121
|
-
const now = this.now();
|
|
122
|
-
const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
123
|
-
if (starvedChore >= 0)
|
|
124
|
-
return this.pending.splice(starvedChore, 1)[0];
|
|
125
|
-
// A receiver's bulk drain is still realtime work, but its slow lane is the
|
|
126
|
-
// progress boundary for that delivery. Preserve its established turn
|
|
127
|
-
// before refilling later fast realtime arrivals; this does not let a
|
|
128
|
-
// chore overtake any realtime work.
|
|
129
|
-
const realtime = this.nextRealtimeIndex();
|
|
130
|
-
if (realtime >= 0)
|
|
131
|
-
return this.pending.splice(realtime, 1)[0];
|
|
132
|
-
return this.pending.shift();
|
|
133
|
-
}
|
|
134
|
-
nextRealtimeIndex() {
|
|
135
|
-
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
136
|
-
return slowRealtime >= 0
|
|
137
|
-
? slowRealtime
|
|
138
|
-
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
139
|
-
}
|
|
140
|
-
nextFastRealtime() {
|
|
141
|
-
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
142
|
-
}
|
|
143
|
-
hasStarvedChore() {
|
|
144
|
-
const now = this.now();
|
|
145
|
-
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
146
|
-
}
|
|
147
|
-
canStartFastRealtime(entry) {
|
|
148
|
-
return (!this._handoffActive &&
|
|
149
|
-
this.activeFastRealtime === null &&
|
|
150
|
-
entry.workClass === "realtime" &&
|
|
151
|
-
entry.lane === "fast" &&
|
|
152
|
-
entry.allowFastRealtimeOvertake &&
|
|
153
|
-
this.hasActiveSlowRealtime &&
|
|
154
|
-
!this.hasStarvedChore());
|
|
155
|
-
}
|
|
156
|
-
start(entry) {
|
|
157
|
-
this.active = entry;
|
|
158
|
-
let current;
|
|
159
|
-
try {
|
|
160
|
-
current = this.execute(entry);
|
|
161
|
-
}
|
|
162
|
-
catch (error) {
|
|
163
|
-
current = Promise.reject(error);
|
|
164
|
-
}
|
|
165
|
-
void current.then((outcome) => {
|
|
166
|
-
entry.resolve(outcome);
|
|
167
|
-
this.complete(entry);
|
|
168
|
-
}, (error) => {
|
|
169
|
-
entry.reject(error);
|
|
170
|
-
this.complete(entry);
|
|
171
|
-
}).catch(() => undefined);
|
|
172
|
-
return current;
|
|
173
|
-
}
|
|
174
|
-
complete(entry) {
|
|
175
|
-
if (this.active === entry)
|
|
176
|
-
this.active = null;
|
|
177
|
-
this.drain();
|
|
178
|
-
}
|
|
179
|
-
startFastRealtime(entry) {
|
|
180
|
-
this.activeFastRealtime = entry;
|
|
181
|
-
let current;
|
|
182
|
-
try {
|
|
183
|
-
current = this.execute(entry);
|
|
184
|
-
}
|
|
185
|
-
catch (error) {
|
|
186
|
-
current = Promise.reject(error);
|
|
187
|
-
}
|
|
188
|
-
void current.then((outcome) => {
|
|
189
|
-
entry.resolve(outcome);
|
|
190
|
-
this.completeFastRealtime(entry);
|
|
191
|
-
}, (error) => {
|
|
192
|
-
entry.reject(error);
|
|
193
|
-
this.completeFastRealtime(entry);
|
|
194
|
-
});
|
|
195
|
-
return current;
|
|
196
|
-
}
|
|
197
|
-
completeFastRealtime(entry) {
|
|
198
|
-
if (this.activeFastRealtime === entry)
|
|
199
|
-
this.activeFastRealtime = null;
|
|
200
|
-
this.drain();
|
|
201
|
-
}
|
|
202
|
-
execute(entry) {
|
|
203
|
-
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
204
|
-
}
|
|
205
|
-
async checkpoint(entry) {
|
|
206
|
-
if (this.stopped ||
|
|
207
|
-
entry.workClass !== "chore" ||
|
|
208
|
-
this.active !== entry ||
|
|
209
|
-
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
210
|
-
return false;
|
|
211
|
-
}
|
|
212
|
-
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
213
|
-
// the sole full-pass continuation and will resume after this bounded burst.
|
|
214
|
-
const deferredAt = this.now();
|
|
215
|
-
this.active = null;
|
|
216
|
-
this._handoffActive = true;
|
|
217
|
-
let handedOff = false;
|
|
218
|
-
try {
|
|
219
|
-
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
220
|
-
const realtimeIndex = this.nextRealtimeIndex();
|
|
221
|
-
if (realtimeIndex < 0)
|
|
222
|
-
break;
|
|
223
|
-
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
224
|
-
handedOff = true;
|
|
225
|
-
this.active = realtime;
|
|
226
|
-
try {
|
|
227
|
-
realtime.resolve(await this.execute(realtime));
|
|
228
|
-
}
|
|
229
|
-
catch (error) {
|
|
230
|
-
realtime.reject(error);
|
|
231
|
-
}
|
|
232
|
-
finally {
|
|
233
|
-
if (this.active === realtime)
|
|
234
|
-
this.active = null;
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
finally {
|
|
239
|
-
this._handoffActive = false;
|
|
240
|
-
if (!this.stopped)
|
|
241
|
-
this.active = entry;
|
|
242
|
-
}
|
|
243
|
-
return handedOff;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
33
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
247
34
|
/**
|
|
248
35
|
* Area storage is AUTO by default and only applies after durable cutover.
|
|
@@ -722,7 +509,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
722
509
|
const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
|
|
723
510
|
const monotonicNow = deps.monotonicNow ?? (() => performance.now());
|
|
724
511
|
const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
725
|
-
const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
|
|
726
512
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
727
513
|
// An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
|
|
728
514
|
// one). When it is omitted, the interval is resolved fresh each cycle from
|
|
@@ -886,27 +672,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
886
672
|
resolveNoOperationLockUsers = resolve;
|
|
887
673
|
});
|
|
888
674
|
};
|
|
889
|
-
const
|
|
890
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
891
|
-
const reportedAppliedUpserts = new Set();
|
|
892
|
-
const recordDurableAppliedUpserts = (upserts) => {
|
|
893
|
-
for (const applied of upserts) {
|
|
894
|
-
const key = `${applied.relativePath}\u0000${applied.contentHash}`;
|
|
895
|
-
if (reportedAppliedUpserts.has(key))
|
|
896
|
-
continue;
|
|
897
|
-
reportedAppliedUpserts.add(key);
|
|
898
|
-
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
899
|
-
}
|
|
900
|
-
};
|
|
901
|
-
const invokePass = () => {
|
|
902
|
-
if (!runPassSupportsCooperativeCheckpoint)
|
|
903
|
-
return runPass(passArgvForRun);
|
|
904
|
-
// This callback is emitted by the production runner after journal
|
|
905
|
-
// repair/seed and before fanout. It starts ingress early, but never
|
|
906
|
-
// admits a second durable writer: queued realtime still waits for the
|
|
907
|
-
// existing scheduler's cooperative hand-off checkpoint.
|
|
908
|
-
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
909
|
-
};
|
|
675
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow") => {
|
|
910
676
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
911
677
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
912
678
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -918,7 +684,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
918
684
|
heapCheckpoint?.();
|
|
919
685
|
if (typeof outcome === "number")
|
|
920
686
|
return;
|
|
921
|
-
|
|
687
|
+
for (const applied of outcome.result?.appliedUpserts ?? []) {
|
|
688
|
+
rememberAppliedUpsert(applied.relativePath, applied.contentHash);
|
|
689
|
+
}
|
|
922
690
|
};
|
|
923
691
|
// Pre-admit boundary observation.
|
|
924
692
|
heapCheckpoint?.();
|
|
@@ -927,31 +695,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
927
695
|
// its keyed journal delta and per-object mutation are independent from the
|
|
928
696
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
929
697
|
// Never bypass the lock when no slow pass is active.
|
|
930
|
-
if (
|
|
931
|
-
// A full chore is paused at a caller-declared durable checkpoint. Keep
|
|
932
|
-
// the root lock held by this process, but lend it to exactly one
|
|
933
|
-
// realtime pass; the suspended chore cannot mutate until this returns.
|
|
934
|
-
workClass === "realtime" &&
|
|
935
|
-
passScheduler.handoffActive &&
|
|
936
|
-
processOwnsOperationLock) {
|
|
937
|
-
acquireOperationLockUser();
|
|
938
|
-
try {
|
|
939
|
-
prepare?.();
|
|
940
|
-
const outcome = await invokePass();
|
|
941
|
-
recordAppliedUpserts(outcome);
|
|
942
|
-
return outcome;
|
|
943
|
-
}
|
|
944
|
-
finally {
|
|
945
|
-
releaseOperationLockUser();
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
698
|
if (lane === "fast" &&
|
|
949
699
|
processOwnsOperationLock &&
|
|
950
700
|
coordinatorWholeMutationChores === 0) {
|
|
951
701
|
acquireOperationLockUser();
|
|
952
702
|
try {
|
|
953
703
|
prepare?.();
|
|
954
|
-
const outcome = await
|
|
704
|
+
const outcome = await runPass(passArgvForRun);
|
|
955
705
|
recordAppliedUpserts(outcome);
|
|
956
706
|
return outcome;
|
|
957
707
|
}
|
|
@@ -971,7 +721,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
971
721
|
acquireOperationLockUser();
|
|
972
722
|
}
|
|
973
723
|
prepare?.();
|
|
974
|
-
const outcome = await
|
|
724
|
+
const outcome = await runPass(passArgvForRun);
|
|
975
725
|
recordAppliedUpserts(outcome);
|
|
976
726
|
return outcome;
|
|
977
727
|
}
|
|
@@ -1005,7 +755,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1005
755
|
}
|
|
1006
756
|
try {
|
|
1007
757
|
prepare?.();
|
|
1008
|
-
const outcome = await
|
|
758
|
+
const outcome = await runPass(passArgvForRun);
|
|
1009
759
|
recordAppliedUpserts(outcome);
|
|
1010
760
|
return outcome;
|
|
1011
761
|
}
|
|
@@ -1028,18 +778,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1028
778
|
throw err;
|
|
1029
779
|
}
|
|
1030
780
|
};
|
|
1031
|
-
const runTaskWithLock = async (task
|
|
1032
|
-
if (workClass === "realtime" &&
|
|
1033
|
-
passScheduler.handoffActive &&
|
|
1034
|
-
processOwnsOperationLock) {
|
|
1035
|
-
acquireOperationLockUser();
|
|
1036
|
-
try {
|
|
1037
|
-
return await task();
|
|
1038
|
-
}
|
|
1039
|
-
finally {
|
|
1040
|
-
releaseOperationLockUser();
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
781
|
+
const runTaskWithLock = async (task) => {
|
|
1043
782
|
try {
|
|
1044
783
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
1045
784
|
timeoutSec: 0,
|
|
@@ -1133,15 +872,102 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1133
872
|
heapCheckpoint?.();
|
|
1134
873
|
}
|
|
1135
874
|
};
|
|
875
|
+
let activeSlowPass = null;
|
|
876
|
+
let activeFastPass = null;
|
|
877
|
+
const pendingSlowPasses = [];
|
|
878
|
+
const pendingFastPasses = [];
|
|
1136
879
|
const resolveStoppedQueue = () => {
|
|
1137
|
-
|
|
880
|
+
for (const queue of [pendingFastPasses, pendingSlowPasses]) {
|
|
881
|
+
while (queue.length > 0)
|
|
882
|
+
queue.shift()?.resolve(0);
|
|
883
|
+
}
|
|
1138
884
|
};
|
|
1139
|
-
const
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
885
|
+
const drainQueuedPasses = () => {
|
|
886
|
+
if (stopped) {
|
|
887
|
+
resolveStoppedQueue();
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
// A queued slow pass is the bulk-drain progress boundary. Admit it before
|
|
891
|
+
// refilling the realtime queue; otherwise a steady stream of fast work
|
|
892
|
+
// perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
|
|
893
|
+
// cannot issue another receive.
|
|
894
|
+
if (activeSlowPass === null &&
|
|
895
|
+
activeFastPass === null &&
|
|
896
|
+
pendingSlowPasses.length > 0) {
|
|
897
|
+
const slow = pendingSlowPasses.shift();
|
|
898
|
+
if (slow) {
|
|
899
|
+
const current = startGuardedTask(slow.task, "slow");
|
|
900
|
+
void current.then(slow.resolve, slow.reject);
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
if (activeFastPass === null) {
|
|
904
|
+
const fast = pendingFastPasses.shift();
|
|
905
|
+
if (fast) {
|
|
906
|
+
const current = startGuardedTask(fast.task, "fast");
|
|
907
|
+
void current.then(fast.resolve, fast.reject);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
// A fast pass that acquired the operation lock normally (rather than
|
|
911
|
+
// bypassing an in-process slow holder) still owns the cross-process
|
|
912
|
+
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
913
|
+
// overlap an already-lock-owning slow pass.
|
|
914
|
+
if (activeSlowPass === null && activeFastPass === null) {
|
|
915
|
+
const slow = pendingSlowPasses.shift();
|
|
916
|
+
if (slow) {
|
|
917
|
+
const current = startGuardedTask(slow.task, "slow");
|
|
918
|
+
void current.then(slow.resolve, slow.reject);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
};
|
|
922
|
+
const startGuardedTask = (task, lane) => {
|
|
923
|
+
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
924
|
+
// a watcher/scheduler callback before returning its promise. Without this
|
|
925
|
+
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
926
|
+
// second lock contender.
|
|
927
|
+
const placeholder = Promise.resolve(0);
|
|
928
|
+
if (lane === "fast")
|
|
929
|
+
activeFastPass = placeholder;
|
|
930
|
+
else
|
|
931
|
+
activeSlowPass = placeholder;
|
|
932
|
+
let current;
|
|
933
|
+
try {
|
|
934
|
+
current = stopped ? placeholder : task();
|
|
935
|
+
}
|
|
936
|
+
catch (err) {
|
|
937
|
+
current = Promise.reject(err);
|
|
938
|
+
}
|
|
939
|
+
if (lane === "fast")
|
|
940
|
+
activeFastPass = current;
|
|
941
|
+
else
|
|
942
|
+
activeSlowPass = current;
|
|
943
|
+
void current
|
|
944
|
+
.finally(() => {
|
|
945
|
+
if (lane === "fast" && activeFastPass === current) {
|
|
946
|
+
activeFastPass = null;
|
|
947
|
+
drainQueuedPasses();
|
|
948
|
+
}
|
|
949
|
+
else if (lane === "slow" && activeSlowPass === current) {
|
|
950
|
+
activeSlowPass = null;
|
|
951
|
+
drainQueuedPasses();
|
|
952
|
+
}
|
|
953
|
+
})
|
|
954
|
+
.catch(() => undefined);
|
|
955
|
+
return current;
|
|
956
|
+
};
|
|
957
|
+
const runGuardedTask = (task, lane = "slow") => {
|
|
958
|
+
const active = lane === "fast"
|
|
959
|
+
? activeFastPass
|
|
960
|
+
: activeSlowPass ?? activeFastPass;
|
|
961
|
+
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
962
|
+
if (active === null && queued.length === 0) {
|
|
963
|
+
return startGuardedTask(task, lane);
|
|
964
|
+
}
|
|
965
|
+
return new Promise((resolve, reject) => {
|
|
966
|
+
queued.push({ task, lane, resolve, reject });
|
|
967
|
+
drainQueuedPasses();
|
|
968
|
+
});
|
|
969
|
+
};
|
|
970
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
1145
971
|
/*
|
|
1146
972
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
1147
973
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1333,7 +1159,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1333
1159
|
});
|
|
1334
1160
|
},
|
|
1335
1161
|
onCutOver: () => {
|
|
1336
|
-
releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1337
1162
|
parallelCoordinatorEnabled = refreshCoordinatorEnabled();
|
|
1338
1163
|
if (parallelCoordinatorEnabled && coordinator === null) {
|
|
1339
1164
|
coordinator = createCoordinator();
|
|
@@ -1420,11 +1245,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1420
1245
|
let drainedUntrackedDeletes = 0;
|
|
1421
1246
|
const pendingWatcherPaths = new Map();
|
|
1422
1247
|
const pendingWatcherChanges = new Map();
|
|
1423
|
-
// A warm-up scan observes filesystem state over time. Keep an ingress
|
|
1424
|
-
// sequence so its older observations cannot overwrite a live watcher event
|
|
1425
|
-
// that arrived while the scan was yielding.
|
|
1426
|
-
const pendingWatcherChangeSequences = new Map();
|
|
1427
|
-
let nextPendingWatcherChangeSequence = 0;
|
|
1428
1248
|
let pendingWatcherOriginalBatch = null;
|
|
1429
1249
|
// Every coalesced watcher batch gets one durable in-process generation.
|
|
1430
1250
|
// A wake can cause the scheduler to revisit a drained batch, but an event
|
|
@@ -1448,11 +1268,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1448
1268
|
const paths = new Map();
|
|
1449
1269
|
const changes = new Map();
|
|
1450
1270
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1451
|
-
const
|
|
1452
|
-
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1271
|
+
const route = routeChangeToTarget(relativePath);
|
|
1453
1272
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1454
1273
|
continue;
|
|
1455
1274
|
paths.set(absolutePath, relativePath);
|
|
1275
|
+
const change = batch.changes?.get(absolutePath);
|
|
1456
1276
|
if (change)
|
|
1457
1277
|
changes.set(absolutePath, change);
|
|
1458
1278
|
}
|
|
@@ -1460,7 +1280,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1460
1280
|
return null;
|
|
1461
1281
|
return { ...batch, paths, changes };
|
|
1462
1282
|
};
|
|
1463
|
-
const addPendingWatcherChange = (changedRelPath, batch
|
|
1283
|
+
const addPendingWatcherChange = (changedRelPath, batch) => {
|
|
1464
1284
|
if (batch) {
|
|
1465
1285
|
const selectedBatch = selectWatchBatch(batch);
|
|
1466
1286
|
if (!selectedBatch)
|
|
@@ -1473,30 +1293,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1473
1293
|
return false;
|
|
1474
1294
|
}
|
|
1475
1295
|
}
|
|
1296
|
+
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1476
1297
|
if (batch) {
|
|
1477
|
-
const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
|
|
1478
|
-
const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
|
|
1479
|
-
return previousSequence === undefined ||
|
|
1480
|
-
options?.preserveChangesAfter === undefined ||
|
|
1481
|
-
previousSequence <= options.preserveChangesAfter;
|
|
1482
|
-
});
|
|
1483
|
-
if (entries.length === 0)
|
|
1484
|
-
return false;
|
|
1485
|
-
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1486
1298
|
pendingWatcherOriginalBatch =
|
|
1487
1299
|
pendingWatcherPaths.size === 0 &&
|
|
1488
1300
|
!pendingWatcherBareChange &&
|
|
1489
1301
|
!pendingWatcherOverflowed &&
|
|
1490
|
-
!batch.overflowed
|
|
1491
|
-
entries.length === batch.paths.size
|
|
1302
|
+
!batch.overflowed
|
|
1492
1303
|
? batch
|
|
1493
1304
|
: null;
|
|
1494
|
-
for (const [absolutePath, relativePath] of entries) {
|
|
1305
|
+
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1495
1306
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1496
1307
|
const change = batch.changes?.get(absolutePath);
|
|
1497
1308
|
if (change)
|
|
1498
1309
|
pendingWatcherChanges.set(absolutePath, change);
|
|
1499
|
-
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1500
1310
|
}
|
|
1501
1311
|
if (batch.overflowed) {
|
|
1502
1312
|
pendingWatcherOverflowed = true;
|
|
@@ -1513,7 +1323,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1513
1323
|
}
|
|
1514
1324
|
return true;
|
|
1515
1325
|
}
|
|
1516
|
-
pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
|
|
1517
1326
|
if (changedRelPath) {
|
|
1518
1327
|
pendingWatcherOriginalBatch = null;
|
|
1519
1328
|
// Codec boundary: `changedRelPath` is a canonical vault key (the
|
|
@@ -1522,7 +1331,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1522
1331
|
// point at the impossible colon path on Windows.
|
|
1523
1332
|
const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
|
|
1524
1333
|
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
1525
|
-
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1526
1334
|
// A bare path signal carries no delete intent; drop any change record
|
|
1527
1335
|
// a prior batch left for this path so the stale intent cannot leak
|
|
1528
1336
|
// into the coalesced batch (#227).
|
|
@@ -1564,7 +1372,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1564
1372
|
const generation = pendingWatcherBatchGeneration;
|
|
1565
1373
|
pendingWatcherPaths.clear();
|
|
1566
1374
|
pendingWatcherChanges.clear();
|
|
1567
|
-
pendingWatcherChangeSequences.clear();
|
|
1568
1375
|
pendingWatcherOriginalBatch = null;
|
|
1569
1376
|
pendingWatcherBareChange = false;
|
|
1570
1377
|
pendingWatcherWarmupCatchup = false;
|
|
@@ -1583,7 +1390,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1583
1390
|
pendingWatcherOriginalBatch = null;
|
|
1584
1391
|
for (const [absolutePath, relativePath] of paths.entries()) {
|
|
1585
1392
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
1586
|
-
pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
|
|
1587
1393
|
}
|
|
1588
1394
|
};
|
|
1589
1395
|
/**
|
|
@@ -1664,8 +1470,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1664
1470
|
}
|
|
1665
1471
|
return null;
|
|
1666
1472
|
};
|
|
1667
|
-
const journalCoordinates = (relativePath
|
|
1668
|
-
const route = routeChangeToTarget(relativePath
|
|
1473
|
+
const journalCoordinates = (relativePath) => {
|
|
1474
|
+
const route = routeChangeToTarget(relativePath);
|
|
1669
1475
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
1670
1476
|
return null;
|
|
1671
1477
|
if (route.kind === "personal") {
|
|
@@ -1700,8 +1506,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1700
1506
|
declaredSize >= 0) {
|
|
1701
1507
|
return declaredSize;
|
|
1702
1508
|
}
|
|
1703
|
-
const size = journalRows(
|
|
1704
|
-
.get(coordinates.key)?.size;
|
|
1509
|
+
const size = journalRows(readJournal(coordinates.slug)).get(coordinates.key)?.size;
|
|
1705
1510
|
return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
|
|
1706
1511
|
};
|
|
1707
1512
|
/**
|
|
@@ -1748,8 +1553,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1748
1553
|
return session.get(coordinates.key) === undefined;
|
|
1749
1554
|
};
|
|
1750
1555
|
const captureLocalDeleteSnapshots = (relativePath, kind) => {
|
|
1751
|
-
const coordinates = journalCoordinates(relativePath
|
|
1752
|
-
const route = routeChangeToTarget(relativePath
|
|
1556
|
+
const coordinates = journalCoordinates(relativePath);
|
|
1557
|
+
const route = routeChangeToTarget(relativePath);
|
|
1753
1558
|
if (!coordinates || !route)
|
|
1754
1559
|
return [];
|
|
1755
1560
|
// Desktop workspace Off toggles are authoritative before journal I/O.
|
|
@@ -1870,8 +1675,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1870
1675
|
* it scopes a push or pull for the affected batch.
|
|
1871
1676
|
*/
|
|
1872
1677
|
const captureQueuedLocalDeleteSnapshots = async (relativePath, kind, isCurrent) => {
|
|
1873
|
-
const coordinates = journalCoordinates(relativePath
|
|
1874
|
-
const route = routeChangeToTarget(relativePath
|
|
1678
|
+
const coordinates = journalCoordinates(relativePath);
|
|
1679
|
+
const route = routeChangeToTarget(relativePath);
|
|
1875
1680
|
if (!coordinates || !route)
|
|
1876
1681
|
return [];
|
|
1877
1682
|
if (route.kind === "company" &&
|
|
@@ -1965,7 +1770,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1965
1770
|
}
|
|
1966
1771
|
return snapshots;
|
|
1967
1772
|
};
|
|
1968
|
-
const warmupJournalPaths =
|
|
1773
|
+
const warmupJournalPaths = () => {
|
|
1774
|
+
const paths = [];
|
|
1969
1775
|
const journals = listJournals();
|
|
1970
1776
|
const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1971
1777
|
for (const { slug, journal } of journals) {
|
|
@@ -1998,9 +1804,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1998
1804
|
// ~12.8k of them per runner start after a transcript prune.
|
|
1999
1805
|
if (route.kind === "personal" && isPersonalVaultDeletionExcluded(key))
|
|
2000
1806
|
continue;
|
|
2001
|
-
|
|
1807
|
+
paths.push(route.kind === "personal" ? key : `companies/${route.slug}/${key}`);
|
|
2002
1808
|
}
|
|
2003
1809
|
}
|
|
1810
|
+
return paths;
|
|
2004
1811
|
};
|
|
2005
1812
|
const publishedContentHashLookup = (relativePath) => {
|
|
2006
1813
|
const coordinates = journalCoordinates(relativePath);
|
|
@@ -2013,34 +1820,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2013
1820
|
return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
|
|
2014
1821
|
};
|
|
2015
1822
|
const prepareWatcherChanges = (changes) => {
|
|
2016
|
-
const keysByJournal = new Map();
|
|
2017
|
-
const rememberKey = (slug, key) => {
|
|
2018
|
-
const keys = keysByJournal.get(slug) ?? new Set();
|
|
2019
|
-
keys.add(key);
|
|
2020
|
-
keysByJournal.set(slug, keys);
|
|
2021
|
-
};
|
|
2022
|
-
for (const [absolutePath, change] of changes.entries()) {
|
|
2023
|
-
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
2024
|
-
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
2025
|
-
rememberKey(snapshot.journalSlug, snapshot.journalPath);
|
|
2026
|
-
}
|
|
2027
|
-
continue;
|
|
2028
|
-
}
|
|
2029
|
-
const relativePath = path
|
|
2030
|
-
.relative(hqRoot, absolutePath)
|
|
2031
|
-
.split(path.sep)
|
|
2032
|
-
.join("/");
|
|
2033
|
-
const coordinates = journalCoordinates(relativePath);
|
|
2034
|
-
if (coordinates)
|
|
2035
|
-
rememberKey(coordinates.slug, coordinates.key);
|
|
2036
|
-
}
|
|
2037
1823
|
const journals = new Map();
|
|
2038
1824
|
const dirty = new Set();
|
|
2039
1825
|
const journalFor = (slug) => {
|
|
2040
1826
|
const loaded = journals.get(slug);
|
|
2041
1827
|
if (loaded)
|
|
2042
1828
|
return loaded;
|
|
2043
|
-
const journal =
|
|
1829
|
+
const journal = readJournal(slug);
|
|
2044
1830
|
journals.set(slug, journal);
|
|
2045
1831
|
return journal;
|
|
2046
1832
|
};
|
|
@@ -2098,8 +1884,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2098
1884
|
const groupBatchByRoute = (batch) => {
|
|
2099
1885
|
const grouped = new Map();
|
|
2100
1886
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
2101
|
-
const
|
|
2102
|
-
const route = routeChangeToTarget(relativePath, change?.kind);
|
|
1887
|
+
const route = routeChangeToTarget(relativePath);
|
|
2103
1888
|
if (!route)
|
|
2104
1889
|
continue;
|
|
2105
1890
|
const key = routeKey(route);
|
|
@@ -2112,6 +1897,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2112
1897
|
deleteScopeRoots: [],
|
|
2113
1898
|
};
|
|
2114
1899
|
existing.paths.set(absolutePath, relativePath);
|
|
1900
|
+
const change = batch.changes?.get(absolutePath);
|
|
2115
1901
|
if (change)
|
|
2116
1902
|
existing.changes.set(absolutePath, change);
|
|
2117
1903
|
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
@@ -2195,7 +1981,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2195
1981
|
}
|
|
2196
1982
|
return false;
|
|
2197
1983
|
};
|
|
2198
|
-
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false
|
|
1984
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
2199
1985
|
if (batch)
|
|
2200
1986
|
await resolveDeleteSnapshotCaptures(batch);
|
|
2201
1987
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
@@ -2224,11 +2010,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2224
2010
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2225
2011
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2226
2012
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2227
|
-
: undefined, lane
|
|
2013
|
+
: undefined, lane);
|
|
2228
2014
|
})
|
|
2229
2015
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2230
2016
|
? () => prepareWatcherChanges(group.changes)
|
|
2231
|
-
: undefined, lane
|
|
2017
|
+
: undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
|
|
2232
2018
|
const result = execution.result;
|
|
2233
2019
|
if (execution.accepted && passExitCode(result) === 0) {
|
|
2234
2020
|
const refusedPaths = new Set(typeof result === "number"
|
|
@@ -2298,8 +2084,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2298
2084
|
? "fast"
|
|
2299
2085
|
: "slow";
|
|
2300
2086
|
const execution = parallelCoordinatorEnabled
|
|
2301
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane
|
|
2302
|
-
: { result: await runGuarded(pullArgv, undefined, lane
|
|
2087
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
|
|
2088
|
+
: { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
|
|
2303
2089
|
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
2304
2090
|
if (exit !== 0 && worstExit === 0)
|
|
2305
2091
|
worstExit = exit;
|
|
@@ -2504,13 +2290,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2504
2290
|
return;
|
|
2505
2291
|
}
|
|
2506
2292
|
if (lanes?.fast &&
|
|
2507
|
-
|
|
2293
|
+
activeSlowPass !== null) {
|
|
2508
2294
|
const generation = ++nextWatcherBatchGeneration;
|
|
2509
2295
|
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
2510
2296
|
// of its contracts here: only selected routes may run, and V2 sees
|
|
2511
2297
|
// the same dirty signal as the compatibility drain.
|
|
2512
2298
|
realtimeState.scheduler?.signal("watcher");
|
|
2513
|
-
void runScopedDrain(lanes.fast, generation, true
|
|
2299
|
+
void runScopedDrain(lanes.fast, generation, true).then((exit) => {
|
|
2514
2300
|
if (exit === 0)
|
|
2515
2301
|
return;
|
|
2516
2302
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
@@ -2559,36 +2345,29 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2559
2345
|
watcherRediscoveryRetries = 0;
|
|
2560
2346
|
process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
|
|
2561
2347
|
const startedAt = Date.now();
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
pathsFound: catchup.batch.paths.size,
|
|
2578
|
-
scannedPaths: catchup.scannedPaths,
|
|
2579
|
-
elapsedMs: Date.now() - startedAt,
|
|
2580
|
-
queued,
|
|
2581
|
-
}) + "\n");
|
|
2582
|
-
if (queued)
|
|
2583
|
-
interruptWait();
|
|
2584
|
-
}
|
|
2585
|
-
catch (err) {
|
|
2586
|
-
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2587
|
-
// A scan failure must not silently extend the warm-up blind spot.
|
|
2588
|
-
pendingWatcherWarmupCatchupFailed = true;
|
|
2348
|
+
try {
|
|
2349
|
+
const catchup = watcher.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
|
|
2350
|
+
if (!catchup)
|
|
2351
|
+
return;
|
|
2352
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch);
|
|
2353
|
+
pendingWatcherWarmupCatchup ||= queued;
|
|
2354
|
+
process.stderr.write(JSON.stringify({
|
|
2355
|
+
type: "watcher-warmup-catchup",
|
|
2356
|
+
reason: "watch-set-ready",
|
|
2357
|
+
pathsFound: catchup.batch.paths.size,
|
|
2358
|
+
scannedPaths: catchup.scannedPaths,
|
|
2359
|
+
elapsedMs: Date.now() - startedAt,
|
|
2360
|
+
queued,
|
|
2361
|
+
}) + "\n");
|
|
2362
|
+
if (queued)
|
|
2589
2363
|
interruptWait();
|
|
2590
|
-
|
|
2591
|
-
|
|
2364
|
+
}
|
|
2365
|
+
catch (err) {
|
|
2366
|
+
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
2367
|
+
// A scan failure must not silently extend the warm-up blind spot.
|
|
2368
|
+
pendingWatcherWarmupCatchupFailed = true;
|
|
2369
|
+
interruptWait();
|
|
2370
|
+
}
|
|
2592
2371
|
};
|
|
2593
2372
|
if (watcher.onReady)
|
|
2594
2373
|
watcher.onReady(reportWatcherReady);
|
|
@@ -2636,7 +2415,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2636
2415
|
// Submit every route in this lane before awaiting any result. In
|
|
2637
2416
|
// particular, a later route's fast receipt must never wait for an
|
|
2638
2417
|
// earlier route's slow pull to finish.
|
|
2639
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane
|
|
2418
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
|
|
2640
2419
|
for (const execution of completed) {
|
|
2641
2420
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2642
2421
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -2690,7 +2469,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2690
2469
|
}
|
|
2691
2470
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
2692
2471
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
2693
|
-
const result = await runGuarded(targetedArgv, undefined, lane
|
|
2472
|
+
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
2694
2473
|
if (passExitCode(result) !== 0) {
|
|
2695
2474
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
2696
2475
|
}
|
|
@@ -2738,14 +2517,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2738
2517
|
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2739
2518
|
await drain();
|
|
2740
2519
|
return 0;
|
|
2741
|
-
}
|
|
2520
|
+
})));
|
|
2742
2521
|
outcome = execution.accepted ? execution.result : 1;
|
|
2743
2522
|
}
|
|
2744
2523
|
else {
|
|
2745
2524
|
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2746
2525
|
await drain();
|
|
2747
2526
|
return 0;
|
|
2748
|
-
}
|
|
2527
|
+
}));
|
|
2749
2528
|
}
|
|
2750
2529
|
}
|
|
2751
2530
|
catch (err) {
|
|
@@ -2941,8 +2720,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2941
2720
|
targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
|
|
2942
2721
|
targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
|
|
2943
2722
|
cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
|
|
2944
|
-
pendingSlowPasses: () =>
|
|
2945
|
-
pendingFastPasses: () =>
|
|
2723
|
+
pendingSlowPasses: () => pendingSlowPasses.length,
|
|
2724
|
+
pendingFastPasses: () => pendingFastPasses.length,
|
|
2946
2725
|
},
|
|
2947
2726
|
eventSyncSizes: () => eventSync?.censusSizes?.(),
|
|
2948
2727
|
watcherSizes: () => watcher?.censusSizes?.(),
|
|
@@ -3292,14 +3071,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3292
3071
|
quarantinedDeleteRoots.delete(root);
|
|
3293
3072
|
}
|
|
3294
3073
|
// Delete snapshot capture needs a fresh keyed view while an episode is
|
|
3295
|
-
// being assembled, but completed watch cycles do not.
|
|
3296
|
-
//
|
|
3297
|
-
//
|
|
3298
|
-
// makes the next scoped callback synchronously re-pack the whole vault.
|
|
3299
|
-
// Keep it for this runner's lifetime; layout/phase changes still close
|
|
3300
|
-
// it in openAreaMigrationIfAuthoritative(), and final shutdown releases
|
|
3301
|
-
// every bridge below.
|
|
3074
|
+
// being assembled, but completed watch cycles do not. Releasing it here
|
|
3075
|
+
// prevents the watcher from retaining another full-vault projection next
|
|
3076
|
+
// to the targeted-pull and area-ledger views.
|
|
3302
3077
|
releaseWatcherJournalStores();
|
|
3078
|
+
releaseCachedAreaJournalMigrations();
|
|
3303
3079
|
const exitCode = passExitCode(result);
|
|
3304
3080
|
if (exitCode === runtime.authRequiredPassExit) {
|
|
3305
3081
|
// The pass already emitted auth-error. Hand the sentinel to the
|
|
@@ -3334,19 +3110,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3334
3110
|
: "") +
|
|
3335
3111
|
"; staying alive, retrying next poll\n");
|
|
3336
3112
|
}
|
|
3337
|
-
else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
3338
|
-
// HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
|
|
3339
|
-
// company directory names, which may not be memberships. The runner
|
|
3340
|
-
// has already emitted the resolution error; terminating here would
|
|
3341
|
-
// restart the outpost watcher every 60 seconds for each such route.
|
|
3342
|
-
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
3343
|
-
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
3344
|
-
}
|
|
3345
3113
|
else if (exitCode !== 0) {
|
|
3346
|
-
|
|
3347
|
-
// pass. The error was emitted by the runner; terminating the daemon
|
|
3348
|
-
// recreates the startup blind window and strands newer watcher work.
|
|
3349
|
-
process.stderr.write(`hq-sync-runner: watch pass failed (exit=${exitCode}); staying alive, retrying next poll\n`);
|
|
3114
|
+
return exitCode;
|
|
3350
3115
|
}
|
|
3351
3116
|
const completedPostRepairPass = !enqueuedFullChore && (typeof result === "number"
|
|
3352
3117
|
? exitCode === 0
|
|
@@ -3450,10 +3215,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3450
3215
|
releaseWatcherJournalStores();
|
|
3451
3216
|
releaseCachedAreaJournalMigrations();
|
|
3452
3217
|
await shutdown();
|
|
3453
|
-
// Coordinator shutdown waits for any active pass before the retained
|
|
3454
|
-
// projections are dropped. A shutdown must never evict a store that a
|
|
3455
|
-
// final scoped writer is still refreshing or committing through.
|
|
3456
|
-
disposeScopedJournalStores();
|
|
3457
3218
|
detachSignal?.();
|
|
3458
3219
|
}
|
|
3459
3220
|
}
|