@indigoai-us/hq-cloud 6.16.52-alpha.0 → 6.16.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +23 -1
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +92 -8
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +36 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -6
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +1 -30
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -55
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -48
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +116 -311
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -33
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -34
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +79 -802
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1454 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +627 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +37 -57
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +93 -87
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +0 -12
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -9
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +10 -70
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -259
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +12 -37
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +76 -291
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.js +4 -4
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
|
@@ -10,11 +10,11 @@ import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
|
10
10
|
import { acquireOperationLock, withOperationLock, withOperationLockSync, OperationLockedError, OPERATION_LOCKED_EXIT, } from "../operation-lock.js";
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
|
-
import { PARTIAL_SYNC_EXIT
|
|
13
|
+
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
14
14
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
|
-
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice,
|
|
17
|
+
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, readJournal, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
18
18
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
19
19
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
20
20
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
@@ -30,219 +30,6 @@ import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
|
30
30
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
31
31
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
32
32
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
33
|
-
/** A deferred full chore must get the next admission after this long. */
|
|
34
|
-
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
35
|
-
/**
|
|
36
|
-
* One durable-state owner at a time for a full chore. Realtime work never runs
|
|
37
|
-
* beside that chore: it explicitly awaits `checkpoint()` after a completed
|
|
38
|
-
* durable unit, then resumes once the hand-off is finished. A separately
|
|
39
|
-
* tracked, explicitly opted-in watcher fast pass may use the pre-existing
|
|
40
|
-
* scoped fast-over-slow exception, but never while a chore owns durable state.
|
|
41
|
-
*/
|
|
42
|
-
export class CooperativePassScheduler {
|
|
43
|
-
now;
|
|
44
|
-
starvationCapMs;
|
|
45
|
-
pending = [];
|
|
46
|
-
active = null;
|
|
47
|
-
// Scoped fast realtime work has the established in-process lock exception
|
|
48
|
-
// while a scoped slow realtime upload is in flight. Keep that slot separate
|
|
49
|
-
// from `active`: a chore still has exactly one durable-state owner.
|
|
50
|
-
activeFastRealtime = null;
|
|
51
|
-
stopped = false;
|
|
52
|
-
_handoffActive = false;
|
|
53
|
-
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS) {
|
|
54
|
-
this.now = now;
|
|
55
|
-
this.starvationCapMs = starvationCapMs;
|
|
56
|
-
}
|
|
57
|
-
get handoffActive() {
|
|
58
|
-
return this._handoffActive;
|
|
59
|
-
}
|
|
60
|
-
get hasActiveChore() {
|
|
61
|
-
return this.active?.workClass === "chore";
|
|
62
|
-
}
|
|
63
|
-
get hasActiveSlowRealtime() {
|
|
64
|
-
return this.active?.workClass === "realtime" && this.active.lane === "slow";
|
|
65
|
-
}
|
|
66
|
-
pendingLaneCount(lane) {
|
|
67
|
-
return this.pending.filter((entry) => entry.lane === lane).length;
|
|
68
|
-
}
|
|
69
|
-
submit(task, workClass, lane = "slow", allowFastRealtimeOvertake = false) {
|
|
70
|
-
if (this.stopped)
|
|
71
|
-
return Promise.resolve(0);
|
|
72
|
-
const directEntry = {
|
|
73
|
-
task,
|
|
74
|
-
workClass,
|
|
75
|
-
lane,
|
|
76
|
-
allowFastRealtimeOvertake,
|
|
77
|
-
// An immediately admitted task cannot starve, so avoid sampling the
|
|
78
|
-
// cadence clock merely to stamp it. This preserves the loop's deadline
|
|
79
|
-
// accounting and keeps the timestamp meaningful only for real queues.
|
|
80
|
-
enqueuedAt: 0,
|
|
81
|
-
resolve: () => undefined,
|
|
82
|
-
reject: () => undefined,
|
|
83
|
-
};
|
|
84
|
-
if (this.canStartFastRealtime(directEntry)) {
|
|
85
|
-
return this.startFastRealtime(directEntry);
|
|
86
|
-
}
|
|
87
|
-
if (this.active === null &&
|
|
88
|
-
this.activeFastRealtime === null &&
|
|
89
|
-
this.pending.length === 0) {
|
|
90
|
-
return this.start(directEntry);
|
|
91
|
-
}
|
|
92
|
-
return new Promise((resolve, reject) => {
|
|
93
|
-
const entry = { ...directEntry, enqueuedAt: this.now(), resolve, reject };
|
|
94
|
-
this.pending.push(entry);
|
|
95
|
-
this.drain();
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
stop() {
|
|
99
|
-
this.stopped = true;
|
|
100
|
-
while (this.pending.length > 0)
|
|
101
|
-
this.pending.shift()?.resolve(0);
|
|
102
|
-
}
|
|
103
|
-
drain() {
|
|
104
|
-
if (this.stopped || this.pending.length === 0)
|
|
105
|
-
return;
|
|
106
|
-
if (this.active !== null) {
|
|
107
|
-
const fastRealtime = this.nextFastRealtime();
|
|
108
|
-
if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
|
|
109
|
-
this.pending.splice(this.pending.indexOf(fastRealtime), 1);
|
|
110
|
-
this.startFastRealtime(fastRealtime);
|
|
111
|
-
}
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
if (this.activeFastRealtime !== null)
|
|
115
|
-
return;
|
|
116
|
-
const next = this.takeNext();
|
|
117
|
-
if (next)
|
|
118
|
-
this.start(next);
|
|
119
|
-
}
|
|
120
|
-
takeNext() {
|
|
121
|
-
const now = this.now();
|
|
122
|
-
const starvedChore = this.pending.findIndex((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
123
|
-
if (starvedChore >= 0)
|
|
124
|
-
return this.pending.splice(starvedChore, 1)[0];
|
|
125
|
-
// A receiver's bulk drain is still realtime work, but its slow lane is the
|
|
126
|
-
// progress boundary for that delivery. Preserve its established turn
|
|
127
|
-
// before refilling later fast realtime arrivals; this does not let a
|
|
128
|
-
// chore overtake any realtime work.
|
|
129
|
-
const realtime = this.nextRealtimeIndex();
|
|
130
|
-
if (realtime >= 0)
|
|
131
|
-
return this.pending.splice(realtime, 1)[0];
|
|
132
|
-
return this.pending.shift();
|
|
133
|
-
}
|
|
134
|
-
nextRealtimeIndex() {
|
|
135
|
-
const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
|
|
136
|
-
return slowRealtime >= 0
|
|
137
|
-
? slowRealtime
|
|
138
|
-
: this.pending.findIndex((entry) => entry.workClass === "realtime");
|
|
139
|
-
}
|
|
140
|
-
nextFastRealtime() {
|
|
141
|
-
return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
|
|
142
|
-
}
|
|
143
|
-
hasStarvedChore() {
|
|
144
|
-
const now = this.now();
|
|
145
|
-
return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
|
|
146
|
-
}
|
|
147
|
-
canStartFastRealtime(entry) {
|
|
148
|
-
return (!this._handoffActive &&
|
|
149
|
-
this.activeFastRealtime === null &&
|
|
150
|
-
entry.workClass === "realtime" &&
|
|
151
|
-
entry.lane === "fast" &&
|
|
152
|
-
entry.allowFastRealtimeOvertake &&
|
|
153
|
-
this.hasActiveSlowRealtime &&
|
|
154
|
-
!this.hasStarvedChore());
|
|
155
|
-
}
|
|
156
|
-
start(entry) {
|
|
157
|
-
this.active = entry;
|
|
158
|
-
let current;
|
|
159
|
-
try {
|
|
160
|
-
current = this.execute(entry);
|
|
161
|
-
}
|
|
162
|
-
catch (error) {
|
|
163
|
-
current = Promise.reject(error);
|
|
164
|
-
}
|
|
165
|
-
void current.then((outcome) => {
|
|
166
|
-
entry.resolve(outcome);
|
|
167
|
-
this.complete(entry);
|
|
168
|
-
}, (error) => {
|
|
169
|
-
entry.reject(error);
|
|
170
|
-
this.complete(entry);
|
|
171
|
-
}).catch(() => undefined);
|
|
172
|
-
return current;
|
|
173
|
-
}
|
|
174
|
-
complete(entry) {
|
|
175
|
-
if (this.active === entry)
|
|
176
|
-
this.active = null;
|
|
177
|
-
this.drain();
|
|
178
|
-
}
|
|
179
|
-
startFastRealtime(entry) {
|
|
180
|
-
this.activeFastRealtime = entry;
|
|
181
|
-
let current;
|
|
182
|
-
try {
|
|
183
|
-
current = this.execute(entry);
|
|
184
|
-
}
|
|
185
|
-
catch (error) {
|
|
186
|
-
current = Promise.reject(error);
|
|
187
|
-
}
|
|
188
|
-
void current.then((outcome) => {
|
|
189
|
-
entry.resolve(outcome);
|
|
190
|
-
this.completeFastRealtime(entry);
|
|
191
|
-
}, (error) => {
|
|
192
|
-
entry.reject(error);
|
|
193
|
-
this.completeFastRealtime(entry);
|
|
194
|
-
});
|
|
195
|
-
return current;
|
|
196
|
-
}
|
|
197
|
-
completeFastRealtime(entry) {
|
|
198
|
-
if (this.activeFastRealtime === entry)
|
|
199
|
-
this.activeFastRealtime = null;
|
|
200
|
-
this.drain();
|
|
201
|
-
}
|
|
202
|
-
execute(entry) {
|
|
203
|
-
return entry.task({ checkpoint: () => this.checkpoint(entry) });
|
|
204
|
-
}
|
|
205
|
-
async checkpoint(entry) {
|
|
206
|
-
if (this.stopped ||
|
|
207
|
-
entry.workClass !== "chore" ||
|
|
208
|
-
this.active !== entry ||
|
|
209
|
-
!this.pending.some((pending) => pending.workClass === "realtime")) {
|
|
210
|
-
return false;
|
|
211
|
-
}
|
|
212
|
-
// `entry` is suspended at a caller-declared durable boundary. It remains
|
|
213
|
-
// the sole full-pass continuation and will resume after this bounded burst.
|
|
214
|
-
const deferredAt = this.now();
|
|
215
|
-
this.active = null;
|
|
216
|
-
this._handoffActive = true;
|
|
217
|
-
let handedOff = false;
|
|
218
|
-
try {
|
|
219
|
-
while (!this.stopped && this.now() - deferredAt < this.starvationCapMs) {
|
|
220
|
-
const realtimeIndex = this.nextRealtimeIndex();
|
|
221
|
-
if (realtimeIndex < 0)
|
|
222
|
-
break;
|
|
223
|
-
const realtime = this.pending.splice(realtimeIndex, 1)[0];
|
|
224
|
-
handedOff = true;
|
|
225
|
-
this.active = realtime;
|
|
226
|
-
try {
|
|
227
|
-
realtime.resolve(await this.execute(realtime));
|
|
228
|
-
}
|
|
229
|
-
catch (error) {
|
|
230
|
-
realtime.reject(error);
|
|
231
|
-
}
|
|
232
|
-
finally {
|
|
233
|
-
if (this.active === realtime)
|
|
234
|
-
this.active = null;
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
finally {
|
|
239
|
-
this._handoffActive = false;
|
|
240
|
-
if (!this.stopped)
|
|
241
|
-
this.active = entry;
|
|
242
|
-
}
|
|
243
|
-
return handedOff;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
33
|
const MAX_FULL_RECONCILE_REASONS = 7;
|
|
247
34
|
/**
|
|
248
35
|
* Area storage is AUTO by default and only applies after durable cutover.
|
|
@@ -722,7 +509,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
722
509
|
const idleHeartbeatIntervalMs = deps.idleHeartbeatIntervalMs ?? WATCH_IDLE_HEARTBEAT_INTERVAL_MS;
|
|
723
510
|
const monotonicNow = deps.monotonicNow ?? (() => performance.now());
|
|
724
511
|
const runPass = deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
725
|
-
const runPassSupportsCooperativeCheckpoint = deps.runPass === undefined || deps.runPassSupportsCooperativeCheckpoint === true;
|
|
726
512
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
727
513
|
// An explicit --poll-remote-ms pins a fixed interval (menubar/CLI still set
|
|
728
514
|
// one). When it is omitted, the interval is resolved fresh each cycle from
|
|
@@ -886,11 +672,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
886
672
|
resolveNoOperationLockUsers = resolve;
|
|
887
673
|
});
|
|
888
674
|
};
|
|
889
|
-
const
|
|
890
|
-
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint) => {
|
|
891
|
-
const invokePass = () => checkpoint !== undefined && runPassSupportsCooperativeCheckpoint
|
|
892
|
-
? runPass(passArgvForRun, checkpoint)
|
|
893
|
-
: runPass(passArgvForRun);
|
|
675
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow") => {
|
|
894
676
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
895
677
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
896
678
|
// synchronous stretch starves the interval timer, so these boundary
|
|
@@ -913,31 +695,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
913
695
|
// its keyed journal delta and per-object mutation are independent from the
|
|
914
696
|
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
915
697
|
// Never bypass the lock when no slow pass is active.
|
|
916
|
-
if (
|
|
917
|
-
// A full chore is paused at a caller-declared durable checkpoint. Keep
|
|
918
|
-
// the root lock held by this process, but lend it to exactly one
|
|
919
|
-
// realtime pass; the suspended chore cannot mutate until this returns.
|
|
920
|
-
workClass === "realtime" &&
|
|
921
|
-
passScheduler.handoffActive &&
|
|
922
|
-
processOwnsOperationLock) {
|
|
923
|
-
acquireOperationLockUser();
|
|
924
|
-
try {
|
|
925
|
-
prepare?.();
|
|
926
|
-
const outcome = await invokePass();
|
|
927
|
-
recordAppliedUpserts(outcome);
|
|
928
|
-
return outcome;
|
|
929
|
-
}
|
|
930
|
-
finally {
|
|
931
|
-
releaseOperationLockUser();
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
698
|
if (lane === "fast" &&
|
|
935
699
|
processOwnsOperationLock &&
|
|
936
700
|
coordinatorWholeMutationChores === 0) {
|
|
937
701
|
acquireOperationLockUser();
|
|
938
702
|
try {
|
|
939
703
|
prepare?.();
|
|
940
|
-
const outcome = await
|
|
704
|
+
const outcome = await runPass(passArgvForRun);
|
|
941
705
|
recordAppliedUpserts(outcome);
|
|
942
706
|
return outcome;
|
|
943
707
|
}
|
|
@@ -957,7 +721,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
957
721
|
acquireOperationLockUser();
|
|
958
722
|
}
|
|
959
723
|
prepare?.();
|
|
960
|
-
const outcome = await
|
|
724
|
+
const outcome = await runPass(passArgvForRun);
|
|
961
725
|
recordAppliedUpserts(outcome);
|
|
962
726
|
return outcome;
|
|
963
727
|
}
|
|
@@ -991,7 +755,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
991
755
|
}
|
|
992
756
|
try {
|
|
993
757
|
prepare?.();
|
|
994
|
-
const outcome = await
|
|
758
|
+
const outcome = await runPass(passArgvForRun);
|
|
995
759
|
recordAppliedUpserts(outcome);
|
|
996
760
|
return outcome;
|
|
997
761
|
}
|
|
@@ -1014,18 +778,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1014
778
|
throw err;
|
|
1015
779
|
}
|
|
1016
780
|
};
|
|
1017
|
-
const runTaskWithLock = async (task
|
|
1018
|
-
if (workClass === "realtime" &&
|
|
1019
|
-
passScheduler.handoffActive &&
|
|
1020
|
-
processOwnsOperationLock) {
|
|
1021
|
-
acquireOperationLockUser();
|
|
1022
|
-
try {
|
|
1023
|
-
return await task();
|
|
1024
|
-
}
|
|
1025
|
-
finally {
|
|
1026
|
-
releaseOperationLockUser();
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
781
|
+
const runTaskWithLock = async (task) => {
|
|
1029
782
|
try {
|
|
1030
783
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
1031
784
|
timeoutSec: 0,
|
|
@@ -1119,15 +872,102 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1119
872
|
heapCheckpoint?.();
|
|
1120
873
|
}
|
|
1121
874
|
};
|
|
875
|
+
let activeSlowPass = null;
|
|
876
|
+
let activeFastPass = null;
|
|
877
|
+
const pendingSlowPasses = [];
|
|
878
|
+
const pendingFastPasses = [];
|
|
1122
879
|
const resolveStoppedQueue = () => {
|
|
1123
|
-
|
|
880
|
+
for (const queue of [pendingFastPasses, pendingSlowPasses]) {
|
|
881
|
+
while (queue.length > 0)
|
|
882
|
+
queue.shift()?.resolve(0);
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
const drainQueuedPasses = () => {
|
|
886
|
+
if (stopped) {
|
|
887
|
+
resolveStoppedQueue();
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
// A queued slow pass is the bulk-drain progress boundary. Admit it before
|
|
891
|
+
// refilling the realtime queue; otherwise a steady stream of fast work
|
|
892
|
+
// perpetually leapfrogs a receiver's bulk batch and its SQS poll loop
|
|
893
|
+
// cannot issue another receive.
|
|
894
|
+
if (activeSlowPass === null &&
|
|
895
|
+
activeFastPass === null &&
|
|
896
|
+
pendingSlowPasses.length > 0) {
|
|
897
|
+
const slow = pendingSlowPasses.shift();
|
|
898
|
+
if (slow) {
|
|
899
|
+
const current = startGuardedTask(slow.task, "slow");
|
|
900
|
+
void current.then(slow.resolve, slow.reject);
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
if (activeFastPass === null) {
|
|
904
|
+
const fast = pendingFastPasses.shift();
|
|
905
|
+
if (fast) {
|
|
906
|
+
const current = startGuardedTask(fast.task, "fast");
|
|
907
|
+
void current.then(fast.resolve, fast.reject);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
// A fast pass that acquired the operation lock normally (rather than
|
|
911
|
+
// bypassing an in-process slow holder) still owns the cross-process
|
|
912
|
+
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
913
|
+
// overlap an already-lock-owning slow pass.
|
|
914
|
+
if (activeSlowPass === null && activeFastPass === null) {
|
|
915
|
+
const slow = pendingSlowPasses.shift();
|
|
916
|
+
if (slow) {
|
|
917
|
+
const current = startGuardedTask(slow.task, "slow");
|
|
918
|
+
void current.then(slow.resolve, slow.reject);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
1124
921
|
};
|
|
1125
|
-
const
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
922
|
+
const startGuardedTask = (task, lane) => {
|
|
923
|
+
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
924
|
+
// a watcher/scheduler callback before returning its promise. Without this
|
|
925
|
+
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
926
|
+
// second lock contender.
|
|
927
|
+
const placeholder = Promise.resolve(0);
|
|
928
|
+
if (lane === "fast")
|
|
929
|
+
activeFastPass = placeholder;
|
|
930
|
+
else
|
|
931
|
+
activeSlowPass = placeholder;
|
|
932
|
+
let current;
|
|
933
|
+
try {
|
|
934
|
+
current = stopped ? placeholder : task();
|
|
935
|
+
}
|
|
936
|
+
catch (err) {
|
|
937
|
+
current = Promise.reject(err);
|
|
938
|
+
}
|
|
939
|
+
if (lane === "fast")
|
|
940
|
+
activeFastPass = current;
|
|
941
|
+
else
|
|
942
|
+
activeSlowPass = current;
|
|
943
|
+
void current
|
|
944
|
+
.finally(() => {
|
|
945
|
+
if (lane === "fast" && activeFastPass === current) {
|
|
946
|
+
activeFastPass = null;
|
|
947
|
+
drainQueuedPasses();
|
|
948
|
+
}
|
|
949
|
+
else if (lane === "slow" && activeSlowPass === current) {
|
|
950
|
+
activeSlowPass = null;
|
|
951
|
+
drainQueuedPasses();
|
|
952
|
+
}
|
|
953
|
+
})
|
|
954
|
+
.catch(() => undefined);
|
|
955
|
+
return current;
|
|
956
|
+
};
|
|
957
|
+
const runGuardedTask = (task, lane = "slow") => {
|
|
958
|
+
const active = lane === "fast"
|
|
959
|
+
? activeFastPass
|
|
960
|
+
: activeSlowPass ?? activeFastPass;
|
|
961
|
+
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
962
|
+
if (active === null && queued.length === 0) {
|
|
963
|
+
return startGuardedTask(task, lane);
|
|
964
|
+
}
|
|
965
|
+
return new Promise((resolve, reject) => {
|
|
966
|
+
queued.push({ task, lane, resolve, reject });
|
|
967
|
+
drainQueuedPasses();
|
|
968
|
+
});
|
|
969
|
+
};
|
|
970
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
1131
971
|
/*
|
|
1132
972
|
* Area storage and the parallel coordinator default to AUTO. AUTO turns
|
|
1133
973
|
* them on after durable personal-vault cutover; explicit off is the kill
|
|
@@ -1319,7 +1159,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1319
1159
|
});
|
|
1320
1160
|
},
|
|
1321
1161
|
onCutOver: () => {
|
|
1322
|
-
releaseScopedJournalStore(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
1323
1162
|
parallelCoordinatorEnabled = refreshCoordinatorEnabled();
|
|
1324
1163
|
if (parallelCoordinatorEnabled && coordinator === null) {
|
|
1325
1164
|
coordinator = createCoordinator();
|
|
@@ -1667,8 +1506,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1667
1506
|
declaredSize >= 0) {
|
|
1668
1507
|
return declaredSize;
|
|
1669
1508
|
}
|
|
1670
|
-
const size = journalRows(
|
|
1671
|
-
.get(coordinates.key)?.size;
|
|
1509
|
+
const size = journalRows(readJournal(coordinates.slug)).get(coordinates.key)?.size;
|
|
1672
1510
|
return typeof size === "number" && Number.isSafeInteger(size) && size >= 0 ? size : null;
|
|
1673
1511
|
};
|
|
1674
1512
|
/**
|
|
@@ -1982,34 +1820,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1982
1820
|
return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
|
|
1983
1821
|
};
|
|
1984
1822
|
const prepareWatcherChanges = (changes) => {
|
|
1985
|
-
const keysByJournal = new Map();
|
|
1986
|
-
const rememberKey = (slug, key) => {
|
|
1987
|
-
const keys = keysByJournal.get(slug) ?? new Set();
|
|
1988
|
-
keys.add(key);
|
|
1989
|
-
keysByJournal.set(slug, keys);
|
|
1990
|
-
};
|
|
1991
|
-
for (const [absolutePath, change] of changes.entries()) {
|
|
1992
|
-
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
1993
|
-
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
1994
|
-
rememberKey(snapshot.journalSlug, snapshot.journalPath);
|
|
1995
|
-
}
|
|
1996
|
-
continue;
|
|
1997
|
-
}
|
|
1998
|
-
const relativePath = path
|
|
1999
|
-
.relative(hqRoot, absolutePath)
|
|
2000
|
-
.split(path.sep)
|
|
2001
|
-
.join("/");
|
|
2002
|
-
const coordinates = journalCoordinates(relativePath);
|
|
2003
|
-
if (coordinates)
|
|
2004
|
-
rememberKey(coordinates.slug, coordinates.key);
|
|
2005
|
-
}
|
|
2006
1823
|
const journals = new Map();
|
|
2007
1824
|
const dirty = new Set();
|
|
2008
1825
|
const journalFor = (slug) => {
|
|
2009
1826
|
const loaded = journals.get(slug);
|
|
2010
1827
|
if (loaded)
|
|
2011
1828
|
return loaded;
|
|
2012
|
-
const journal =
|
|
1829
|
+
const journal = readJournal(slug);
|
|
2013
1830
|
journals.set(slug, journal);
|
|
2014
1831
|
return journal;
|
|
2015
1832
|
};
|
|
@@ -2164,7 +1981,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2164
1981
|
}
|
|
2165
1982
|
return false;
|
|
2166
1983
|
};
|
|
2167
|
-
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false
|
|
1984
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
2168
1985
|
if (batch)
|
|
2169
1986
|
await resolveDeleteSnapshotCaptures(batch);
|
|
2170
1987
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
@@ -2193,11 +2010,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2193
2010
|
const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
|
|
2194
2011
|
return runGuarded(scopedArgv, admittedChanges.size > 0
|
|
2195
2012
|
? () => prepareWatcherChanges(admittedChanges)
|
|
2196
|
-
: undefined, lane
|
|
2013
|
+
: undefined, lane);
|
|
2197
2014
|
})
|
|
2198
2015
|
: { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
|
|
2199
2016
|
? () => prepareWatcherChanges(group.changes)
|
|
2200
|
-
: undefined, lane
|
|
2017
|
+
: undefined, lane), accepted: true, coordinatorRefusedPaths: [] };
|
|
2201
2018
|
const result = execution.result;
|
|
2202
2019
|
if (execution.accepted && passExitCode(result) === 0) {
|
|
2203
2020
|
const refusedPaths = new Set(typeof result === "number"
|
|
@@ -2267,8 +2084,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2267
2084
|
? "fast"
|
|
2268
2085
|
: "slow";
|
|
2269
2086
|
const execution = parallelCoordinatorEnabled
|
|
2270
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane
|
|
2271
|
-
: { result: await runGuarded(pullArgv, undefined, lane
|
|
2087
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice) => runGuarded(buildTargetedPullArgv(group.route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))
|
|
2088
|
+
: { result: await runGuarded(pullArgv, undefined, lane), accepted: true };
|
|
2272
2089
|
const exit = execution.accepted ? passExitCode(execution.result) : 1;
|
|
2273
2090
|
if (exit !== 0 && worstExit === 0)
|
|
2274
2091
|
worstExit = exit;
|
|
@@ -2473,13 +2290,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2473
2290
|
return;
|
|
2474
2291
|
}
|
|
2475
2292
|
if (lanes?.fast &&
|
|
2476
|
-
|
|
2293
|
+
activeSlowPass !== null) {
|
|
2477
2294
|
const generation = ++nextWatcherBatchGeneration;
|
|
2478
2295
|
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
2479
2296
|
// of its contracts here: only selected routes may run, and V2 sees
|
|
2480
2297
|
// the same dirty signal as the compatibility drain.
|
|
2481
2298
|
realtimeState.scheduler?.signal("watcher");
|
|
2482
|
-
void runScopedDrain(lanes.fast, generation, true
|
|
2299
|
+
void runScopedDrain(lanes.fast, generation, true).then((exit) => {
|
|
2483
2300
|
if (exit === 0)
|
|
2484
2301
|
return;
|
|
2485
2302
|
restorePendingWatcherPaths(lanes.fast.paths, generation);
|
|
@@ -2598,7 +2415,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2598
2415
|
// Submit every route in this lane before awaiting any result. In
|
|
2599
2416
|
// particular, a later route's fast receipt must never wait for an
|
|
2600
2417
|
// earlier route's slow pull to finish.
|
|
2601
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane
|
|
2418
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane))));
|
|
2602
2419
|
for (const execution of completed) {
|
|
2603
2420
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
2604
2421
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -2652,7 +2469,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2652
2469
|
}
|
|
2653
2470
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
2654
2471
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
2655
|
-
const result = await runGuarded(targetedArgv, undefined, lane
|
|
2472
|
+
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
2656
2473
|
if (passExitCode(result) !== 0) {
|
|
2657
2474
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
2658
2475
|
}
|
|
@@ -2700,14 +2517,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2700
2517
|
const execution = await runRealtimePass("wake", "pull", [], "slow", async () => runGuardedTask(() => runTaskWithLock(async () => {
|
|
2701
2518
|
await drain();
|
|
2702
2519
|
return 0;
|
|
2703
|
-
}
|
|
2520
|
+
})));
|
|
2704
2521
|
outcome = execution.accepted ? execution.result : 1;
|
|
2705
2522
|
}
|
|
2706
2523
|
else {
|
|
2707
2524
|
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
2708
2525
|
await drain();
|
|
2709
2526
|
return 0;
|
|
2710
|
-
}
|
|
2527
|
+
}));
|
|
2711
2528
|
}
|
|
2712
2529
|
}
|
|
2713
2530
|
catch (err) {
|
|
@@ -2903,8 +2720,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2903
2720
|
targetedJournalSessions: () => targetedJournalSessionCensusSizes().targetedJournalSessions,
|
|
2904
2721
|
targetedJournalRows: () => targetedJournalSessionCensusSizes().targetedJournalRows,
|
|
2905
2722
|
cachedAreaJournalMigrations: () => journalCacheCensusSizes().cachedAreaJournalMigrations,
|
|
2906
|
-
pendingSlowPasses: () =>
|
|
2907
|
-
pendingFastPasses: () =>
|
|
2723
|
+
pendingSlowPasses: () => pendingSlowPasses.length,
|
|
2724
|
+
pendingFastPasses: () => pendingFastPasses.length,
|
|
2908
2725
|
},
|
|
2909
2726
|
eventSyncSizes: () => eventSync?.censusSizes?.(),
|
|
2910
2727
|
watcherSizes: () => watcher?.censusSizes?.(),
|
|
@@ -3293,14 +3110,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3293
3110
|
: "") +
|
|
3294
3111
|
"; staying alive, retrying next poll\n");
|
|
3295
3112
|
}
|
|
3296
|
-
else if (exitCode === UNRESOLVED_COMPANY_EXIT) {
|
|
3297
|
-
// HQ-SYNC-COMPANY-ROUTE-UID: routed paths are derived from local
|
|
3298
|
-
// company directory names, which may not be memberships. The runner
|
|
3299
|
-
// has already emitted the resolution error; terminating here would
|
|
3300
|
-
// restart the outpost watcher every 60 seconds for each such route.
|
|
3301
|
-
// Keep the watcher alive while preserving the one-shot non-zero code.
|
|
3302
|
-
process.stderr.write("hq-sync-runner: watch pass skipped — unresolved company target; retrying next poll\n");
|
|
3303
|
-
}
|
|
3304
3113
|
else if (exitCode !== 0) {
|
|
3305
3114
|
return exitCode;
|
|
3306
3115
|
}
|
|
@@ -3406,10 +3215,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3406
3215
|
releaseWatcherJournalStores();
|
|
3407
3216
|
releaseCachedAreaJournalMigrations();
|
|
3408
3217
|
await shutdown();
|
|
3409
|
-
// Coordinator shutdown waits for any active pass before the retained
|
|
3410
|
-
// projections are dropped. A shutdown must never evict a store that a
|
|
3411
|
-
// final scoped writer is still refreshing or committing through.
|
|
3412
|
-
disposeScopedJournalStores();
|
|
3413
3218
|
detachSignal?.();
|
|
3414
3219
|
}
|
|
3415
3220
|
}
|