@indigoai-us/hq-cloud 6.2.7 → 6.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +22 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +105 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +262 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts +8 -0
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +222 -198
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +35 -0
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-core.js +14 -2
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-hq-root-guard.test.d.ts +2 -0
- package/dist/cli/rescue-hq-root-guard.test.d.ts.map +1 -0
- package/dist/cli/rescue-hq-root-guard.test.js +176 -0
- package/dist/cli/rescue-hq-root-guard.test.js.map +1 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +39 -16
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.reindex.test.js +15 -2
- package/dist/cli/rescue.reindex.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +3 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/operation-lock.d.ts +100 -0
- package/dist/operation-lock.d.ts.map +1 -0
- package/dist/operation-lock.js +256 -0
- package/dist/operation-lock.js.map +1 -0
- package/dist/operation-lock.test.d.ts +5 -0
- package/dist/operation-lock.test.d.ts.map +1 -0
- package/dist/operation-lock.test.js +140 -0
- package/dist/operation-lock.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +181 -0
- package/dist/sync/event-sync.d.ts.map +1 -0
- package/dist/sync/event-sync.js +316 -0
- package/dist/sync/event-sync.js.map +1 -0
- package/dist/sync/event-sync.test.d.ts +14 -0
- package/dist/sync/event-sync.test.d.ts.map +1 -0
- package/dist/sync/event-sync.test.js +440 -0
- package/dist/sync/event-sync.test.js.map +1 -0
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +323 -0
- package/src/bin/sync-runner.ts +139 -4
- package/src/cli/reindex.test.ts +45 -0
- package/src/cli/reindex.ts +36 -0
- package/src/cli/rescue-core.ts +15 -2
- package/src/cli/rescue-hq-root-guard.test.ts +193 -0
- package/src/cli/rescue.reindex.test.ts +17 -2
- package/src/cli/rescue.ts +40 -15
- package/src/cli/sync.test.ts +2 -1
- package/src/cli/sync.ts +3 -1
- package/src/operation-lock.test.ts +162 -0
- package/src/operation-lock.ts +293 -0
- package/src/sync/event-sync.test.ts +533 -0
- package/src/sync/event-sync.ts +481 -0
- package/test/e2e/sync/cross-tenant-isolation.test.ts +126 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-HQ-root mutual exclusion for the long-running operations
|
|
3
|
+
* (`sync`, `rescue`, `reindex`).
|
|
4
|
+
*
|
|
5
|
+
* Contract:
|
|
6
|
+
* - At most ONE of sync / rescue / reindex runs at a time **per HQ root**.
|
|
7
|
+
* The lock is shared across all three (keyed only by the root, not the
|
|
8
|
+
* command), so e.g. a rescue refuses while a sync holds it. Different HQ
|
|
9
|
+
* roots are fully independent — they hash to different lock files.
|
|
10
|
+
* - The push watcher / watch+event-push runner is EXEMPT: it never calls in
|
|
11
|
+
* here, so it neither takes the lock nor is blocked by it (its targeted
|
|
12
|
+
* in-process push passes are likewise lock-free).
|
|
13
|
+
*
|
|
14
|
+
* ## Where the lock lives — and why
|
|
15
|
+
*
|
|
16
|
+
* `<stateDir>/locks/operation-<hash(canonicalRoot)>.lock`, where
|
|
17
|
+
* `stateDir = $HQ_STATE_DIR || ~/.hq`. This is deliberately NOT inside the HQ
|
|
18
|
+
* root:
|
|
19
|
+
* - It must never round-trip to the cloud. A lock is machine-local, per-run
|
|
20
|
+
* state; syncing it to S3 (and thence to other machines/roots) would be a
|
|
21
|
+
* correctness bug. `~/.hq` is the established machine-local state dir
|
|
22
|
+
* (journals already live there) and is never synced.
|
|
23
|
+
* - `rescue` repairs a possibly-broken HQ root; a lock that depends on the
|
|
24
|
+
* root being healthy is exactly backwards. `~/.hq` is independent of the
|
|
25
|
+
* root's health.
|
|
26
|
+
* - Keying the filename by a hash of the *canonical* root path makes the
|
|
27
|
+
* lock per-root and prevents leakage across roots, while keeping the path
|
|
28
|
+
* short and filesystem-safe.
|
|
29
|
+
*
|
|
30
|
+
* ## Atomicity, liveness, takeover
|
|
31
|
+
*
|
|
32
|
+
* - Acquisition uses `open(…, "wx")` (O_CREAT | O_EXCL) — an atomic
|
|
33
|
+
* create-if-absent. Exactly one racer can create the file; the loser sees
|
|
34
|
+
* EEXIST and re-evaluates.
|
|
35
|
+
* - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
|
|
36
|
+
* EEXIST we test the recorded PID with `process.kill(pid, 0)`:
|
|
37
|
+
* * ESRCH → the holder is gone (crashed / killed -9 / stale file) →
|
|
38
|
+
* reclaim the lock.
|
|
39
|
+
* * EPERM → the PID exists but is owned by another user → treat as ALIVE
|
|
40
|
+
* (conservative: refuse rather than risk two concurrent ops).
|
|
41
|
+
* * success → alive → refuse fast with {@link OperationLockedError}
|
|
42
|
+
* naming the holding command + PID.
|
|
43
|
+
* - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
|
|
44
|
+
* the original holder crashed and the OS later handed its PID to an
|
|
45
|
+
* unrelated process, we conservatively read that as "still held" and
|
|
46
|
+
* refuse. We accept that false-busy over the far worse false-free, and
|
|
47
|
+
* record `startedAt`/`command` so an operator can diagnose a wedged lock.
|
|
48
|
+
*
|
|
49
|
+
* ## Release
|
|
50
|
+
*
|
|
51
|
+
* - Normal exit: the `with*` wrappers release in a `finally`.
|
|
52
|
+
* - Signals (SIGINT/SIGTERM): a one-time handler releases every held lock,
|
|
53
|
+
* then re-raises the default disposition so exit status is unchanged.
|
|
54
|
+
* - Hard crash (SIGKILL / power loss): nothing runs, but the stale-PID
|
|
55
|
+
* takeover above reclaims the lock on the next attempt.
|
|
56
|
+
* - `process.on("exit")`: a final best-effort synchronous unlink.
|
|
57
|
+
*
|
|
58
|
+
* ## Escape hatch
|
|
59
|
+
*
|
|
60
|
+
* `HQ_DISABLE_OP_LOCK=1` makes acquisition a no-op (returns a handle whose
|
|
61
|
+
* release does nothing). For emergencies and for callers that manage
|
|
62
|
+
* exclusion themselves; documented, off by default.
|
|
63
|
+
*/
|
|
64
|
+
import * as crypto from "crypto";
|
|
65
|
+
import * as fs from "fs";
|
|
66
|
+
import * as os from "os";
|
|
67
|
+
import * as path from "path";
|
|
68
|
+
/** Process exit code used when an operation is refused because the lock is held. */
|
|
69
|
+
export const OPERATION_LOCKED_EXIT = 17;
|
|
70
|
+
/** Thrown by `acquireOperationLock` when a LIVE holder owns the lock. */
|
|
71
|
+
export class OperationLockedError extends Error {
|
|
72
|
+
holder;
|
|
73
|
+
attempted;
|
|
74
|
+
constructor(holder, attempted) {
|
|
75
|
+
super(`Refusing to start "${attempted}": another HQ operation is already ` +
|
|
76
|
+
`running for this HQ root — "${holder.command}" (pid ${holder.pid}, ` +
|
|
77
|
+
`started ${holder.startedAt}). Wait for it to finish, or stop that ` +
|
|
78
|
+
`process, then retry.`);
|
|
79
|
+
this.holder = holder;
|
|
80
|
+
this.attempted = attempted;
|
|
81
|
+
this.name = "OperationLockedError";
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function stateDir() {
|
|
85
|
+
return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
|
|
86
|
+
}
|
|
87
|
+
/** Absolute lock path for a given HQ root. Exported for tests. */
|
|
88
|
+
export function lockPathFor(hqRoot) {
|
|
89
|
+
const canon = path.resolve(hqRoot);
|
|
90
|
+
const key = crypto.createHash("sha1").update(canon).digest("hex").slice(0, 16);
|
|
91
|
+
return path.join(stateDir(), "locks", `operation-${key}.lock`);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Is `pid` a live process? `kill(pid, 0)` sends no signal; it only probes.
|
|
95
|
+
* ESRCH → no such process (dead/stale). EPERM → exists but not ours → ALIVE
|
|
96
|
+
* (conservative). Anything else → assume alive rather than risk a double-run.
|
|
97
|
+
*/
|
|
98
|
+
function pidAlive(pid) {
|
|
99
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
100
|
+
return false;
|
|
101
|
+
try {
|
|
102
|
+
process.kill(pid, 0);
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
const code = err?.code;
|
|
107
|
+
if (code === "ESRCH")
|
|
108
|
+
return false;
|
|
109
|
+
return true; // EPERM (exists) or unknown → treat as alive
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function readLockInfo(p) {
|
|
113
|
+
try {
|
|
114
|
+
const parsed = JSON.parse(fs.readFileSync(p, "utf8"));
|
|
115
|
+
if (parsed && typeof parsed.pid === "number" && typeof parsed.command === "string") {
|
|
116
|
+
return parsed;
|
|
117
|
+
}
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
// ── Process-wide release plumbing ──────────────────────────────────────────
|
|
125
|
+
// Track every lock this process currently holds so the signal/exit hooks can
|
|
126
|
+
// release all of them. The hooks are installed exactly once.
|
|
127
|
+
const heldLocks = new Set();
|
|
128
|
+
let hooksInstalled = false;
|
|
129
|
+
function unlinkIfOwned(p) {
|
|
130
|
+
// Only remove a lock whose recorded pid is THIS process — never clobber a
|
|
131
|
+
// lock another process took over after a (hypothetical) reclaim race.
|
|
132
|
+
const info = readLockInfo(p);
|
|
133
|
+
if (info && info.pid === process.pid) {
|
|
134
|
+
try {
|
|
135
|
+
fs.unlinkSync(p);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
/* already gone — fine */
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function installHooksOnce() {
|
|
143
|
+
if (hooksInstalled)
|
|
144
|
+
return;
|
|
145
|
+
hooksInstalled = true;
|
|
146
|
+
process.on("exit", () => {
|
|
147
|
+
for (const h of heldLocks)
|
|
148
|
+
unlinkIfOwned(h.path);
|
|
149
|
+
});
|
|
150
|
+
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
|
151
|
+
process.on(sig, () => {
|
|
152
|
+
for (const h of heldLocks)
|
|
153
|
+
unlinkIfOwned(h.path);
|
|
154
|
+
// Re-raise with the default disposition so the exit status is the normal
|
|
155
|
+
// signal status (and a second Ctrl-C still works). Removing our listener
|
|
156
|
+
// first avoids recursing back into this handler.
|
|
157
|
+
process.removeAllListeners(sig);
|
|
158
|
+
process.kill(process.pid, sig);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
function makeHandle(p, info) {
|
|
163
|
+
const handle = {
|
|
164
|
+
path: p,
|
|
165
|
+
info,
|
|
166
|
+
release() {
|
|
167
|
+
heldLocks.delete(handle);
|
|
168
|
+
unlinkIfOwned(p);
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
heldLocks.add(handle);
|
|
172
|
+
installHooksOnce();
|
|
173
|
+
return handle;
|
|
174
|
+
}
|
|
175
|
+
const NOOP_HANDLE_BASE = { release() { } };
|
|
176
|
+
/**
|
|
177
|
+
* Acquire the per-root operation lock for `command`. Returns a {@link LockHandle}
|
|
178
|
+
* on success; throws {@link OperationLockedError} when a live holder owns it.
|
|
179
|
+
* Reclaims a stale lock (dead holder) transparently.
|
|
180
|
+
*/
|
|
181
|
+
export function acquireOperationLock(hqRoot, command) {
|
|
182
|
+
if (process.env.HQ_DISABLE_OP_LOCK === "1") {
|
|
183
|
+
const info = {
|
|
184
|
+
pid: process.pid,
|
|
185
|
+
command,
|
|
186
|
+
startedAt: new Date().toISOString(),
|
|
187
|
+
hqRoot: path.resolve(hqRoot),
|
|
188
|
+
};
|
|
189
|
+
return { ...NOOP_HANDLE_BASE, path: "", info };
|
|
190
|
+
}
|
|
191
|
+
const p = lockPathFor(hqRoot);
|
|
192
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
193
|
+
const info = {
|
|
194
|
+
pid: process.pid,
|
|
195
|
+
command,
|
|
196
|
+
startedAt: new Date().toISOString(),
|
|
197
|
+
hqRoot: path.resolve(hqRoot),
|
|
198
|
+
};
|
|
199
|
+
const payload = JSON.stringify(info, null, 2);
|
|
200
|
+
// Bounded retry: each iteration is one atomic create attempt. EEXIST against
|
|
201
|
+
// a stale holder reclaims and retries; EEXIST against a live holder refuses.
|
|
202
|
+
const MAX_ATTEMPTS = 5;
|
|
203
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
204
|
+
let fd;
|
|
205
|
+
try {
|
|
206
|
+
fd = fs.openSync(p, "wx"); // O_CREAT | O_EXCL — atomic
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
if (err?.code !== "EEXIST")
|
|
210
|
+
throw err;
|
|
211
|
+
const holder = readLockInfo(p);
|
|
212
|
+
if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
|
|
213
|
+
throw new OperationLockedError(holder, command);
|
|
214
|
+
}
|
|
215
|
+
// Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
|
|
216
|
+
try {
|
|
217
|
+
fs.unlinkSync(p);
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
/* someone else reclaimed it first; the next openSync re-evaluates */
|
|
221
|
+
}
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
try {
|
|
225
|
+
fs.writeSync(fd, payload);
|
|
226
|
+
}
|
|
227
|
+
finally {
|
|
228
|
+
fs.closeSync(fd);
|
|
229
|
+
}
|
|
230
|
+
return makeHandle(p, info);
|
|
231
|
+
}
|
|
232
|
+
// Pathological churn (another process reclaiming in lockstep). Surface it
|
|
233
|
+
// rather than spin forever.
|
|
234
|
+
throw new Error(`Could not acquire HQ operation lock at ${p} after ${MAX_ATTEMPTS} attempts`);
|
|
235
|
+
}
|
|
236
|
+
/** Run `fn` while holding the per-root lock for `command` (async). */
|
|
237
|
+
export async function withOperationLock(hqRoot, command, fn) {
|
|
238
|
+
const handle = acquireOperationLock(hqRoot, command);
|
|
239
|
+
try {
|
|
240
|
+
return await fn();
|
|
241
|
+
}
|
|
242
|
+
finally {
|
|
243
|
+
handle.release();
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/** Run `fn` while holding the per-root lock for `command` (synchronous). */
|
|
247
|
+
export function withOperationLockSync(hqRoot, command, fn) {
|
|
248
|
+
const handle = acquireOperationLock(hqRoot, command);
|
|
249
|
+
try {
|
|
250
|
+
return fn();
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
handle.release();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
//# sourceMappingURL=operation-lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operation-lock.js","sourceRoot":"","sources":["../src/operation-lock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,oFAAoF;AACpF,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAWxC,yEAAyE;AACzE,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAE3B;IACA;IAFlB,YACkB,MAAgB,EAChB,SAAiB;QAEjC,KAAK,CACH,sBAAsB,SAAS,qCAAqC;YAClE,+BAA+B,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,IAAI;YACrE,WAAW,MAAM,CAAC,SAAS,yCAAyC;YACpE,sBAAsB,CACzB,CAAC;QARc,WAAM,GAAN,MAAM,CAAU;QAChB,cAAS,GAAT,SAAS,CAAQ;QAQjC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAWD,SAAS,QAAQ;IACf,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;AACpE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/E,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAI,GAA6B,EAAE,IAAI,CAAC;QAClD,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,IAAI,CAAC,CAAC,6CAA6C;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAa,CAAC;QAClE,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnF,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,6EAA6E;AAC7E,6DAA6D;AAE7D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc,CAAC;AACxC,IAAI,cAAc,GAAG,KAAK,CAAC;AAE3B,SAAS,aAAa,CAAC,CAAS;IAC9B,0EAA0E;IAC1E,sEAAsE;IACtE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,cAAc;QAAE,OAAO;IAC3B,cAAc,GAAG,IAAI,CAAC;IAEtB,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACtB,KAAK,MAAM,CAAC,IAAI,SAAS;YAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QAC3D,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE;YACnB,KAAK,MAAM,CAAC,IAAI,SAAS;gBAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjD,yEAAyE;YACzE,yEAAyE;YACzE,iDAAiD;YACjD,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAAS,EAAE,IAAc;IAC3C,MAAM,MAAM,GAAe;QACzB,IAAI,EAAE,CAAC;QACP,IAAI;QACJ,OAAO;YACL,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,aAAa,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;KACF,CAAC;IACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,gBAAgB,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,gBAAgB,GAAG,EAAE,OAAO,KAAI,CAAC,EAAE,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,OAAe;IAClE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAa;YACrB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO;YACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;SAC7B,CAAC;QACF,OAAO,EAAE,GAAG,gBAAgB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IACjD,CAAC;IAED,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC7B,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAE9C,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,YAAY,GAAG,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,EAAU,CAAC;QACf,IAAI,CAAC;YACH,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,4BAA4B;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,EAAE,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;YACD,uEAAuE;YACvE,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;YACvE,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,IAAI,KAAK,CACb,0CAA0C,CAAC,UAAU,YAAY,WAAW,CAC7E,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,OAAe,EACf,EAAoB;IAEpB,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,qBAAqB,CACnC,MAAc,EACd,OAAe,EACf,EAAW;IAEX,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operation-lock.test.d.ts","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the per-HQ-root operation mutex.
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
5
|
+
import { spawnSync } from "child_process";
|
|
6
|
+
import * as fs from "fs";
|
|
7
|
+
import * as os from "os";
|
|
8
|
+
import * as path from "path";
|
|
9
|
+
import { acquireOperationLock, withOperationLockSync, lockPathFor, OperationLockedError, OPERATION_LOCKED_EXIT, } from "./operation-lock.js";
|
|
10
|
+
/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
|
|
11
|
+
function deadPid() {
|
|
12
|
+
const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
|
|
13
|
+
if (!r.pid)
|
|
14
|
+
throw new Error("could not spawn to obtain a dead pid");
|
|
15
|
+
return r.pid;
|
|
16
|
+
}
|
|
17
|
+
function writeLock(p, info) {
|
|
18
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
19
|
+
const full = {
|
|
20
|
+
pid: 1,
|
|
21
|
+
command: "sync",
|
|
22
|
+
startedAt: new Date(0).toISOString(),
|
|
23
|
+
hqRoot: "/x",
|
|
24
|
+
...info,
|
|
25
|
+
};
|
|
26
|
+
fs.writeFileSync(p, JSON.stringify(full));
|
|
27
|
+
}
|
|
28
|
+
describe("operation-lock", () => {
|
|
29
|
+
let stateDir;
|
|
30
|
+
let rootA;
|
|
31
|
+
let rootB;
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
|
|
34
|
+
process.env.HQ_STATE_DIR = stateDir;
|
|
35
|
+
delete process.env.HQ_DISABLE_OP_LOCK;
|
|
36
|
+
rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
|
|
37
|
+
rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
|
|
38
|
+
});
|
|
39
|
+
afterEach(() => {
|
|
40
|
+
fs.rmSync(stateDir, { recursive: true, force: true });
|
|
41
|
+
fs.rmSync(rootA, { recursive: true, force: true });
|
|
42
|
+
fs.rmSync(rootB, { recursive: true, force: true });
|
|
43
|
+
delete process.env.HQ_STATE_DIR;
|
|
44
|
+
delete process.env.HQ_DISABLE_OP_LOCK;
|
|
45
|
+
});
|
|
46
|
+
it("the lock path is under the state dir, keyed per canonical root", () => {
|
|
47
|
+
const a = lockPathFor(rootA);
|
|
48
|
+
const b = lockPathFor(rootB);
|
|
49
|
+
expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
|
|
50
|
+
expect(a).not.toBe(b); // different roots → different lock files
|
|
51
|
+
// canonical: a trailing-slash variant maps to the same lock
|
|
52
|
+
expect(lockPathFor(rootA + path.sep)).toBe(a);
|
|
53
|
+
});
|
|
54
|
+
it("acquires, writes holder info, and releases (file gone after release)", () => {
|
|
55
|
+
const h = acquireOperationLock(rootA, "sync");
|
|
56
|
+
expect(fs.existsSync(h.path)).toBe(true);
|
|
57
|
+
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
58
|
+
expect(info.pid).toBe(process.pid);
|
|
59
|
+
expect(info.command).toBe("sync");
|
|
60
|
+
h.release();
|
|
61
|
+
expect(fs.existsSync(h.path)).toBe(false);
|
|
62
|
+
});
|
|
63
|
+
it("refuses fast with the holder's command + pid when a LIVE process holds it", () => {
|
|
64
|
+
// Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
|
|
65
|
+
// is always alive and is never our own pid, so kill(1,0) reports alive and
|
|
66
|
+
// the same-process reclaim path does not apply.
|
|
67
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
|
|
68
|
+
expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
|
|
69
|
+
try {
|
|
70
|
+
acquireOperationLock(rootA, "sync");
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
const err = e;
|
|
74
|
+
expect(err.holder.command).toBe("rescue");
|
|
75
|
+
expect(err.holder.pid).toBe(1);
|
|
76
|
+
expect(err.message).toContain("rescue");
|
|
77
|
+
expect(err.message).toContain("pid 1");
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
it("reclaims a stale lock whose holder PID is dead (takeover)", () => {
|
|
81
|
+
const stale = deadPid();
|
|
82
|
+
writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
|
|
83
|
+
// The dead holder must not block us.
|
|
84
|
+
const h = acquireOperationLock(rootA, "rescue");
|
|
85
|
+
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
86
|
+
expect(info.pid).toBe(process.pid); // we took it over
|
|
87
|
+
expect(info.command).toBe("rescue");
|
|
88
|
+
h.release();
|
|
89
|
+
});
|
|
90
|
+
it("reclaims a torn/unreadable lock file", () => {
|
|
91
|
+
const p = lockPathFor(rootA);
|
|
92
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
93
|
+
fs.writeFileSync(p, "{ this is not valid json");
|
|
94
|
+
const h = acquireOperationLock(rootA, "reindex");
|
|
95
|
+
expect(fs.existsSync(h.path)).toBe(true);
|
|
96
|
+
h.release();
|
|
97
|
+
});
|
|
98
|
+
it("different HQ roots are independent — both may hold concurrently", () => {
|
|
99
|
+
const a = acquireOperationLock(rootA, "sync");
|
|
100
|
+
const b = acquireOperationLock(rootB, "rescue"); // must NOT refuse
|
|
101
|
+
expect(fs.existsSync(a.path)).toBe(true);
|
|
102
|
+
expect(fs.existsSync(b.path)).toBe(true);
|
|
103
|
+
expect(a.path).not.toBe(b.path);
|
|
104
|
+
a.release();
|
|
105
|
+
b.release();
|
|
106
|
+
});
|
|
107
|
+
it("the same root is mutually exclusive across different commands", () => {
|
|
108
|
+
// A live sync in ANOTHER process holds the root (pid 1 stands in for it).
|
|
109
|
+
const p = lockPathFor(rootA);
|
|
110
|
+
writeLock(p, { pid: 1, command: "sync" });
|
|
111
|
+
// Neither rescue nor reindex may acquire while that sync holds it.
|
|
112
|
+
expect(() => acquireOperationLock(rootA, "rescue")).toThrowError(OperationLockedError);
|
|
113
|
+
expect(() => acquireOperationLock(rootA, "reindex")).toThrowError(OperationLockedError);
|
|
114
|
+
// Once that sync finishes (its lock is gone), the next command acquires.
|
|
115
|
+
fs.unlinkSync(p);
|
|
116
|
+
const h2 = acquireOperationLock(rootA, "reindex");
|
|
117
|
+
expect(fs.existsSync(h2.path)).toBe(true);
|
|
118
|
+
h2.release();
|
|
119
|
+
});
|
|
120
|
+
it("withOperationLockSync releases even when the body throws", () => {
|
|
121
|
+
const p = lockPathFor(rootA);
|
|
122
|
+
expect(() => withOperationLockSync(rootA, "reindex", () => {
|
|
123
|
+
expect(fs.existsSync(p)).toBe(true); // held during the body
|
|
124
|
+
throw new Error("boom");
|
|
125
|
+
})).toThrow("boom");
|
|
126
|
+
expect(fs.existsSync(p)).toBe(false); // released on the way out
|
|
127
|
+
});
|
|
128
|
+
it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
|
|
129
|
+
process.env.HQ_DISABLE_OP_LOCK = "1";
|
|
130
|
+
// Even with a live holder on record, the escape hatch acquires without error.
|
|
131
|
+
writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
|
|
132
|
+
const h = acquireOperationLock(rootA, "rescue");
|
|
133
|
+
expect(h.path).toBe(""); // no real lock file written
|
|
134
|
+
h.release(); // no-op, no throw
|
|
135
|
+
});
|
|
136
|
+
it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
|
|
137
|
+
expect(OPERATION_LOCKED_EXIT).toBe(17);
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
//# sourceMappingURL=operation-lock.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operation-lock.test.js","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,WAAW,EACX,oBAAoB,EACpB,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AAE7B,yFAAyF;AACzF,SAAS,OAAO;IACd,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,IAAuB;IACnD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,CAAC;QACN,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QACpC,MAAM,EAAE,IAAI;QACZ,GAAG,IAAI;KACR,CAAC;IACF,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,KAAa,CAAC;IAClB,IAAI,KAAa,CAAC;IAElB,UAAU,CAAC,GAAG,EAAE;QACd,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC;QACpC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,yCAAyC;QAChE,4DAA4D;QAC5D,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC9E,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,2EAA2E;QAC3E,2EAA2E;QAC3E,gDAAgD;QAChD,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAyB,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC;QACxB,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/D,qCAAqC;QACrC,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,kBAAkB;QACnE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,0EAA0E;QAC1E,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,mEAAmE;QACnE,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACvF,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACxF,yEAAyE;QACzE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,EAAE,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,EAAE,CAAC,OAAO,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,EAAE,CACV,qBAAqB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YAC5D,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA0B;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,GAAG,CAAC;QACrC,8EAA8E;QAC9E,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACrD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,kBAAkB;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Event-driven sync wiring — Phase 3 of event-driven-sync-menubar
|
|
3
|
+
* (US-017 publish / US-018 receive / US-019 rollout gate).
|
|
4
|
+
*
|
|
5
|
+
* Phases 1–2 built every piece but left them unconnected: the watcher runs
|
|
6
|
+
* targeted *push passes* (S3 upload) but never publishes a PushEvent, and the
|
|
7
|
+
* runner's receiver seam defaults to {@link NoopPushReceiver}. This module is
|
|
8
|
+
* the connective tissue that turns both on for enrolled accounts:
|
|
9
|
+
*
|
|
10
|
+
* - {@link resolveEventSync} — the rollout gate. EXACT-email allowlist
|
|
11
|
+
* (mirrors `resolvePresignTransport`'s shape in sync-runner.ts, but full
|
|
12
|
+
* address, not domain suffix) + `HQ_SYNC_EVENT_SYNC` env override in both
|
|
13
|
+
* directions. ONE gate governs publish AND receive so a device is never
|
|
14
|
+
* publish-only or receive-only in a half-rolled state.
|
|
15
|
+
* - {@link subscribeSyncReceive} — `POST /v1/sync/subscribe` (US-015/US-016):
|
|
16
|
+
* mints the per-device queue and returns `{queueUrl, region, credentials}`,
|
|
17
|
+
* where `credentials` are short-lived STS creds scoped to receive/delete on
|
|
18
|
+
* exactly that queue.
|
|
19
|
+
* - {@link sqsClientFromAwsSdk} — the doc-promised thin adapter from the AWS
|
|
20
|
+
* SDK `SQSClient` to the receiver's narrow {@link SqsClientLike} seam.
|
|
21
|
+
* - {@link createRefreshingSqsClient} — wraps the adapter with credential
|
|
22
|
+
* lifecycle: proactive re-vend before expiry (skew window) + one reactive
|
|
23
|
+
* retry on an expiry-class error. The queue URL is stable across re-vends
|
|
24
|
+
* (same device → same queue, idempotent endpoint); only creds rotate.
|
|
25
|
+
* - {@link startEventSync} — the wiring entry the runner calls from the
|
|
26
|
+
* `--event-push` watch block when the gate is ON. Resolves tenant + device
|
|
27
|
+
* identity, builds the {@link HttpPushTransport} + {@link PushEventEmitter}
|
|
28
|
+
* (publish leg) and the {@link SqsPushReceiver} (receive leg, self-echo
|
|
29
|
+
* filtered), and returns handles. Any startup failure degrades to
|
|
30
|
+
* poll-only — it NEVER takes the daemon down.
|
|
31
|
+
*
|
|
32
|
+
* The 10-minute `--poll-remote-ms` pass remains the correctness backstop for
|
|
33
|
+
* every path here; event delivery is best-effort by design.
|
|
34
|
+
*/
|
|
35
|
+
import { SQSClient } from "@aws-sdk/client-sqs";
|
|
36
|
+
import { HttpPushTransport, type AuthTokenSource } from "./push-transport.js";
|
|
37
|
+
import { type PushReceiver, type SqsClientLike, type SyncEngineFn } from "./push-receiver.js";
|
|
38
|
+
import type { TreeChangeBatch } from "../watcher.js";
|
|
39
|
+
/**
|
|
40
|
+
* Accounts enrolled in event-driven sync (publish + receive).
|
|
41
|
+
*
|
|
42
|
+
* EXACT full-address matching, case-insensitive — NOT a domain suffix. The
|
|
43
|
+
* single-account Phase 3 rollout (2026-06-10) targets the operator's own
|
|
44
|
+
* devices; `xhassaan@getindigo.ai` and `hassaan@getindigo.ai.evil.com` must
|
|
45
|
+
* never match. Broadening later is an entry here (or a domain-set like
|
|
46
|
+
* `PRESIGN_ROLLOUT_DOMAINS` once GA'd).
|
|
47
|
+
*/
|
|
48
|
+
export declare const EVENT_SYNC_ROLLOUT_EMAILS: ReadonlySet<string>;
|
|
49
|
+
/**
|
|
50
|
+
* Decide whether this session runs event-driven sync (publish + receive).
|
|
51
|
+
*
|
|
52
|
+
* Mirrors `resolvePresignTransport` precedence: `HQ_SYNC_EVENT_SYNC`
|
|
53
|
+
* overrides in both directions (`1`/`true`/`yes`/`on` → force on,
|
|
54
|
+
* `0`/`false`/`no`/`off` → force off) so unenrolled testers can exercise it
|
|
55
|
+
* and enrolled accounts can be rolled back without a release. An unset/blank
|
|
56
|
+
* override falls through to the exact-email check; an unrecognized override
|
|
57
|
+
* value is ignored (email check wins).
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveEventSync(email: string | undefined, override: string | undefined): boolean;
|
|
60
|
+
export interface SubscribeSyncCredentials {
|
|
61
|
+
accessKeyId: string;
|
|
62
|
+
secretAccessKey: string;
|
|
63
|
+
sessionToken: string;
|
|
64
|
+
/** ISO8601 expiry of the vended STS credentials. */
|
|
65
|
+
expiration: string;
|
|
66
|
+
}
|
|
67
|
+
export interface SubscribeSyncResponse {
|
|
68
|
+
/** The caller's own per-device queue URL (stable across calls). */
|
|
69
|
+
queueUrl: string;
|
|
70
|
+
/** Region the queue lives in. */
|
|
71
|
+
region: string;
|
|
72
|
+
/** Short-lived creds scoped to receive/delete on exactly this queue. */
|
|
73
|
+
credentials: SubscribeSyncCredentials;
|
|
74
|
+
}
|
|
75
|
+
/** Minimal fetch seam (matches push-transport.ts's FetchLike posture). */
|
|
76
|
+
type FetchLike = (url: string, init: {
|
|
77
|
+
method: string;
|
|
78
|
+
headers: Record<string, string>;
|
|
79
|
+
body: string;
|
|
80
|
+
signal?: AbortSignal;
|
|
81
|
+
}) => Promise<{
|
|
82
|
+
ok: boolean;
|
|
83
|
+
status: number;
|
|
84
|
+
text(): Promise<string>;
|
|
85
|
+
}>;
|
|
86
|
+
/**
|
|
87
|
+
* `POST /v1/sync/subscribe` — provision (idempotently) this device's queue
|
|
88
|
+
* and vend fresh receive credentials. Auth mirrors HttpPushTransport: Bearer
|
|
89
|
+
* token resolved per-call via the supplied source.
|
|
90
|
+
*/
|
|
91
|
+
export declare function subscribeSyncReceive(opts: {
|
|
92
|
+
apiUrl: string;
|
|
93
|
+
authToken: AuthTokenSource;
|
|
94
|
+
deviceId: string;
|
|
95
|
+
timeoutMs?: number;
|
|
96
|
+
fetchImpl?: FetchLike;
|
|
97
|
+
}): Promise<SubscribeSyncResponse>;
|
|
98
|
+
/**
|
|
99
|
+
* Adapt the AWS SDK `SQSClient` to the receiver's narrow {@link SqsClientLike}
|
|
100
|
+
* seam (the doc-promised `sqsClientFromAwsSdk` from push-receiver.ts). The
|
|
101
|
+
* abort signal is forwarded so `dispose()` can cut a 20s long-poll short.
|
|
102
|
+
*/
|
|
103
|
+
export declare function sqsClientFromAwsSdk(client: Pick<SQSClient, "send">): SqsClientLike;
|
|
104
|
+
export interface RefreshingSqsClientOptions {
|
|
105
|
+
/** The initial subscribe response (creds + region + queue URL). */
|
|
106
|
+
initial: SubscribeSyncResponse;
|
|
107
|
+
/** Re-vend: called when creds are near/past expiry. Idempotent server-side. */
|
|
108
|
+
subscribe: () => Promise<SubscribeSyncResponse>;
|
|
109
|
+
/**
|
|
110
|
+
* Build the underlying narrow client from a subscribe response. Default:
|
|
111
|
+
* AWS SDK `SQSClient` via {@link sqsClientFromAwsSdk}. Tests inject a fake.
|
|
112
|
+
*/
|
|
113
|
+
buildSqs?: (resp: SubscribeSyncResponse) => SqsClientLike;
|
|
114
|
+
/** Clock seam (tests). Default `Date.now`. */
|
|
115
|
+
now?: () => number;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* An {@link SqsClientLike} that owns the vended-credential lifecycle:
|
|
119
|
+
*
|
|
120
|
+
* - PROACTIVE: before each call, if the recorded expiry is within the skew
|
|
121
|
+
* window, re-subscribe (re-vend) and rebuild the inner client first.
|
|
122
|
+
* - REACTIVE: if a call still fails with an expiry-class error (clock skew,
|
|
123
|
+
* revocation), re-vend once and retry the call once. Anything else — or a
|
|
124
|
+
* second failure — propagates to the receiver's own backoff/reconnect
|
|
125
|
+
* loop, whose retention-backed redelivery makes the miss recoverable.
|
|
126
|
+
*
|
|
127
|
+
* Concurrent refreshes collapse onto one in-flight subscribe promise.
|
|
128
|
+
*/
|
|
129
|
+
export declare function createRefreshingSqsClient(opts: RefreshingSqsClientOptions): SqsClientLike;
|
|
130
|
+
/** Structured line logger seam — the runner passes its stderr logger. */
|
|
131
|
+
export type EventSyncLog = (message: string) => void;
|
|
132
|
+
export interface StartEventSyncOptions {
|
|
133
|
+
hqRoot: string;
|
|
134
|
+
/** Vault API base URL (the runner's DEFAULT_VAULT_API_URL). */
|
|
135
|
+
apiUrl: string;
|
|
136
|
+
/** Cognito access-token source (getter for long-running daemons). */
|
|
137
|
+
authToken: AuthTokenSource;
|
|
138
|
+
/** This device's stable id (getOrCreateMachineId). */
|
|
139
|
+
deviceId: string;
|
|
140
|
+
/**
|
|
141
|
+
* Resolve the caller's tenant id (canonical person `prs_*` uid). The server
|
|
142
|
+
* rejects publishes whose `originTenantId` mismatches the JWT principal, so
|
|
143
|
+
* this MUST be the same identity the JWT resolves to.
|
|
144
|
+
*/
|
|
145
|
+
resolveTenantId: () => Promise<string>;
|
|
146
|
+
/**
|
|
147
|
+
* The already-routed targeted-pull bridge (the runner's receiverSyncFn,
|
|
148
|
+
* funneled through its runGuarded mutex). Self-echo filtering happens HERE,
|
|
149
|
+
* before this is invoked.
|
|
150
|
+
*/
|
|
151
|
+
syncFn: SyncEngineFn;
|
|
152
|
+
/** Diagnostic logger (one line per lifecycle event). Default: console.error. */
|
|
153
|
+
log?: EventSyncLog;
|
|
154
|
+
subscribe?: (deviceId: string) => Promise<SubscribeSyncResponse>;
|
|
155
|
+
buildSqs?: (resp: SubscribeSyncResponse) => SqsClientLike;
|
|
156
|
+
transport?: HttpPushTransport;
|
|
157
|
+
now?: () => number;
|
|
158
|
+
}
|
|
159
|
+
export interface EventSyncHandles {
|
|
160
|
+
/**
|
|
161
|
+
* Publish PushEvents for a settled change batch. Called by the runner
|
|
162
|
+
* AFTER the targeted push pass succeeds — an event must never announce
|
|
163
|
+
* bytes that are not in S3 yet. Fire-and-forget: failures are logged by
|
|
164
|
+
* the emitter's onError and the cadence poll covers the miss.
|
|
165
|
+
*/
|
|
166
|
+
publishBatch: (batch: TreeChangeBatch) => void;
|
|
167
|
+
/** The live receiver (already started). */
|
|
168
|
+
receiver: PushReceiver;
|
|
169
|
+
/** This device's id — the runner uses it nowhere else, exposed for logs. */
|
|
170
|
+
ownDeviceId: string;
|
|
171
|
+
/** Tear down transport + receiver (runner shutdown path). */
|
|
172
|
+
dispose: () => Promise<void>;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Bring up the publish + receive legs. Returns `null` (poll-only degradation)
|
|
176
|
+
* on ANY startup failure — subscribe 5xx, tenant resolution failure, etc. —
|
|
177
|
+
* after logging the reason. The caller treats `null` as "today's behavior".
|
|
178
|
+
*/
|
|
179
|
+
export declare function startEventSync(opts: StartEventSyncOptions): Promise<EventSyncHandles | null>;
|
|
180
|
+
export {};
|
|
181
|
+
//# sourceMappingURL=event-sync.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-sync.d.ts","sourceRoot":"","sources":["../../src/sync/event-sync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EACL,SAAS,EAGV,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,iBAAiB,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAIrD;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,EAAE,WAAW,CAAC,MAAM,CAExD,CAAC;AAEH;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,GAC3B,OAAO,CAMT;AAID,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,WAAW,EAAE,wBAAwB,CAAC;CACvC;AAED,0EAA0E;AAC1E,KAAK,SAAS,GAAG,CACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,KACE,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;CAAE,CAAC,CAAC;AASvE;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,eAAe,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAqDjC;AAID;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAC9B,aAAa,CA4Bf;AAwBD,MAAM,WAAW,0BAA0B;IACzC,mEAAmE;IACnE,OAAO,EAAE,qBAAqB,CAAC;IAC/B,+EAA+E;IAC/E,SAAS,EAAE,MAAM,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,qBAAqB,KAAK,aAAa,CAAC;IAC1D,8CAA8C;IAC9C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAgBD;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,0BAA0B,GAC/B,aAAa,CA0Cf;AAID,yEAAyE;AACzE,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;AAErD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,SAAS,EAAE,eAAe,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,eAAe,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,gFAAgF;IAChF,GAAG,CAAC,EAAE,YAAY,CAAC;IAEnB,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACjE,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,qBAAqB,KAAK,aAAa,CAAC;IAC1D,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,YAAY,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC;IAC/C,2CAA2C;IAC3C,QAAQ,EAAE,YAAY,CAAC;IACvB,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAsFlC"}
|