@indigoai-us/hq-cloud 6.7.1 → 6.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +33 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +73 -4
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts +11 -0
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +1 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +5 -4
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +20 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +36 -2
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +38 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/operation-lock.d.ts +81 -10
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +177 -27
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +122 -11
- package/dist/operation-lock.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +83 -4
- package/src/bin/sync-runner.ts +39 -1
- package/src/cli/reindex.test.ts +5 -4
- package/src/cli/reindex.ts +12 -1
- package/src/cli/rescue.test.ts +43 -1
- package/src/cli/rescue.ts +48 -2
- package/src/operation-lock.test.ts +147 -10
- package/src/operation-lock.ts +234 -26
package/dist/operation-lock.js
CHANGED
|
@@ -35,11 +35,44 @@
|
|
|
35
35
|
* - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
|
|
36
36
|
* EEXIST we test the recorded PID with `process.kill(pid, 0)`:
|
|
37
37
|
* * ESRCH → the holder is gone (crashed / killed -9 / stale file) →
|
|
38
|
-
* reclaim the lock
|
|
38
|
+
* reclaim the lock IMMEDIATELY (a dead holder never makes us
|
|
39
|
+
* wait).
|
|
39
40
|
* * EPERM → the PID exists but is owned by another user → treat as ALIVE
|
|
40
|
-
* (conservative:
|
|
41
|
-
* * success → alive →
|
|
42
|
-
*
|
|
41
|
+
* (conservative: wait rather than risk two concurrent ops).
|
|
42
|
+
* * success → alive → WAIT for the holder to release, then acquire (see
|
|
43
|
+
* "Waiting" below). The fast-refusal path is still reachable
|
|
44
|
+
* via an explicit timeout / `wait: false`.
|
|
45
|
+
*
|
|
46
|
+
* ## Waiting for a live holder (default behavior)
|
|
47
|
+
*
|
|
48
|
+
* When a LIVE holder owns the lock, acquisition WAITS by default: it polls
|
|
49
|
+
* (~2s) and acquires the instant the holder releases, rather than refusing
|
|
50
|
+
* fast. A single status line is written to stderr the first time we start
|
|
51
|
+
* waiting ("Waiting for <command> (pid N) to finish…"), never per-poll.
|
|
52
|
+
* This is what an interactive `sync` / `rescue` / `reindex` invocation wants —
|
|
53
|
+
* queue behind the running op instead of erroring out.
|
|
54
|
+
*
|
|
55
|
+
* A bounded escape exists for scripts that must not block forever:
|
|
56
|
+
* - `timeoutSec` option, or the `HQ_OP_LOCK_TIMEOUT` env var (seconds).
|
|
57
|
+
* The option wins over the env. After the bound elapses we throw
|
|
58
|
+
* {@link OperationLockedError} (exit 17) with the same clear refusal
|
|
59
|
+
* message as before.
|
|
60
|
+
* - `timeoutSec === 0` (or `HQ_OP_LOCK_TIMEOUT=0`, or `wait: false`) → do
|
|
61
|
+
* not wait at all; refuse immediately. This is the pre-wait behavior.
|
|
62
|
+
* - absent / negative / unparseable → INFINITE wait (the documented
|
|
63
|
+
* default).
|
|
64
|
+
* Stale-PID takeover is unconditional and happens BEFORE any wait — a dead
|
|
65
|
+
* holder is reclaimed at once regardless of the wait config.
|
|
66
|
+
*
|
|
67
|
+
* Ordering / scope caveats:
|
|
68
|
+
* - This is a CROSS-PROCESS mutex keyed on the holder's PID. Two concurrent
|
|
69
|
+
* acquisitions inside the SAME process share a PID, so the same-process
|
|
70
|
+
* reclaim path lets them stomp each other — in-process concurrent acquire
|
|
71
|
+
* is unsupported (the real consumers — sync / rescue / reindex — are
|
|
72
|
+
* separate processes).
|
|
73
|
+
* - When several distinct processes wait on the same lock, the next one to
|
|
74
|
+
* win the O_EXCL race after a free acquires. Order is best-effort, NOT
|
|
75
|
+
* FIFO — do not depend on arrival order.
|
|
43
76
|
* - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
|
|
44
77
|
* the original holder crashed and the OS later handed its PID to an
|
|
45
78
|
* unrelated process, we conservatively read that as "still held" and
|
|
@@ -81,6 +114,58 @@ export class OperationLockedError extends Error {
|
|
|
81
114
|
this.name = "OperationLockedError";
|
|
82
115
|
}
|
|
83
116
|
}
|
|
117
|
+
/** Default poll interval while waiting on a live holder. */
|
|
118
|
+
export const DEFAULT_LOCK_POLL_MS = 2000;
|
|
119
|
+
/** Default status line: a single stderr message naming the holder. */
|
|
120
|
+
function defaultOnWaitStart(holder, attempted) {
|
|
121
|
+
process.stderr.write(`Waiting for "${holder.command}" (pid ${holder.pid}) to finish before ` +
|
|
122
|
+
`starting "${attempted}"… (set HQ_OP_LOCK_TIMEOUT=<secs> to bound the wait)\n`);
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Resolve the effective wait config from explicit options + the
|
|
126
|
+
* `HQ_OP_LOCK_TIMEOUT` env var. Option timeout wins over the env; `wait: false`
|
|
127
|
+
* forces a zero (no-wait) timeout.
|
|
128
|
+
*/
|
|
129
|
+
function resolveWaitConfig(opts) {
|
|
130
|
+
// Parse a seconds value into ms, or null for "absent/infinite". Only a
|
|
131
|
+
// finite, non-negative number counts; everything else (NaN, Infinity, <0)
|
|
132
|
+
// means "no explicit bound".
|
|
133
|
+
const toMs = (sec) => {
|
|
134
|
+
if (sec === undefined)
|
|
135
|
+
return null;
|
|
136
|
+
if (!Number.isFinite(sec) || sec < 0)
|
|
137
|
+
return null;
|
|
138
|
+
return Math.round(sec * 1000);
|
|
139
|
+
};
|
|
140
|
+
let timeoutMs;
|
|
141
|
+
if (opts.timeoutSec !== undefined) {
|
|
142
|
+
timeoutMs = toMs(opts.timeoutSec);
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
const envRaw = process.env.HQ_OP_LOCK_TIMEOUT;
|
|
146
|
+
timeoutMs = envRaw !== undefined && envRaw !== "" ? toMs(Number(envRaw)) : null;
|
|
147
|
+
}
|
|
148
|
+
// `wait: false` is shorthand for a zero-length wait (refuse immediately).
|
|
149
|
+
if (opts.wait === false)
|
|
150
|
+
timeoutMs = 0;
|
|
151
|
+
const pollMs = opts.pollIntervalMs && opts.pollIntervalMs > 0
|
|
152
|
+
? opts.pollIntervalMs
|
|
153
|
+
: DEFAULT_LOCK_POLL_MS;
|
|
154
|
+
return { timeoutMs, pollMs, onWaitStart: opts.onWaitStart ?? defaultOnWaitStart };
|
|
155
|
+
}
|
|
156
|
+
/** Block the current thread for `ms` without busy-spinning (sync consumers). */
|
|
157
|
+
function sleepSync(ms) {
|
|
158
|
+
if (ms <= 0)
|
|
159
|
+
return;
|
|
160
|
+
// Atomics.wait on a private buffer is a clean, CPU-free sleep. The value at
|
|
161
|
+
// index 0 is 0 and nothing ever notifies it, so this always sleeps the full
|
|
162
|
+
// timeout (or less if interrupted) and returns "timed-out".
|
|
163
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
164
|
+
}
|
|
165
|
+
/** Non-blocking sleep for the async consumer. */
|
|
166
|
+
function sleepAsync(ms) {
|
|
167
|
+
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
168
|
+
}
|
|
84
169
|
function stateDir() {
|
|
85
170
|
return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
|
|
86
171
|
}
|
|
@@ -173,21 +258,18 @@ function makeHandle(p, info) {
|
|
|
173
258
|
return handle;
|
|
174
259
|
}
|
|
175
260
|
const NOOP_HANDLE_BASE = { release() { } };
|
|
176
|
-
/**
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
};
|
|
189
|
-
return { ...NOOP_HANDLE_BASE, path: "", info };
|
|
190
|
-
}
|
|
261
|
+
/** No-op handle for the `HQ_DISABLE_OP_LOCK=1` escape hatch. */
|
|
262
|
+
function disabledHandle(hqRoot, command) {
|
|
263
|
+
const info = {
|
|
264
|
+
pid: process.pid,
|
|
265
|
+
command,
|
|
266
|
+
startedAt: new Date().toISOString(),
|
|
267
|
+
hqRoot: path.resolve(hqRoot),
|
|
268
|
+
};
|
|
269
|
+
return { ...NOOP_HANDLE_BASE, path: "", info };
|
|
270
|
+
}
|
|
271
|
+
/** Build the lock payload + ensure the locks dir exists. */
|
|
272
|
+
function prepareLock(hqRoot, command) {
|
|
191
273
|
const p = lockPathFor(hqRoot);
|
|
192
274
|
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
193
275
|
const info = {
|
|
@@ -196,9 +278,19 @@ export function acquireOperationLock(hqRoot, command) {
|
|
|
196
278
|
startedAt: new Date().toISOString(),
|
|
197
279
|
hqRoot: path.resolve(hqRoot),
|
|
198
280
|
};
|
|
199
|
-
|
|
281
|
+
return { p, info, payload: JSON.stringify(info, null, 2) };
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* One acquisition pass. Returns the {@link LockHandle} on success, or
|
|
285
|
+
* `{ busy }` naming the LIVE holder that blocked us (so the caller can decide
|
|
286
|
+
* to wait or refuse). A stale/torn/own-leftover lock is reclaimed in-pass and
|
|
287
|
+
* never reported as busy. Throws only on genuinely pathological churn or a
|
|
288
|
+
* non-EEXIST fs error.
|
|
289
|
+
*/
|
|
290
|
+
function tryAcquireOnce(p, info, payload) {
|
|
200
291
|
// Bounded retry: each iteration is one atomic create attempt. EEXIST against
|
|
201
|
-
// a stale holder reclaims and retries; EEXIST against a live holder
|
|
292
|
+
// a stale holder reclaims and retries; EEXIST against a live holder reports
|
|
293
|
+
// it as busy.
|
|
202
294
|
const MAX_ATTEMPTS = 5;
|
|
203
295
|
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
204
296
|
let fd;
|
|
@@ -210,7 +302,7 @@ export function acquireOperationLock(hqRoot, command) {
|
|
|
210
302
|
throw err;
|
|
211
303
|
const holder = readLockInfo(p);
|
|
212
304
|
if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
|
|
213
|
-
|
|
305
|
+
return { busy: holder };
|
|
214
306
|
}
|
|
215
307
|
// Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
|
|
216
308
|
try {
|
|
@@ -227,15 +319,73 @@ export function acquireOperationLock(hqRoot, command) {
|
|
|
227
319
|
finally {
|
|
228
320
|
fs.closeSync(fd);
|
|
229
321
|
}
|
|
230
|
-
return makeHandle(p, info);
|
|
322
|
+
return { handle: makeHandle(p, info) };
|
|
231
323
|
}
|
|
232
324
|
// Pathological churn (another process reclaiming in lockstep). Surface it
|
|
233
325
|
// rather than spin forever.
|
|
234
326
|
throw new Error(`Could not acquire HQ operation lock at ${p} after ${MAX_ATTEMPTS} attempts`);
|
|
235
327
|
}
|
|
328
|
+
/** ms left until `deadline` (null deadline → never expires). */
|
|
329
|
+
function remainingMs(deadline) {
|
|
330
|
+
return deadline === null ? Infinity : deadline - Date.now();
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Acquire the per-root operation lock for `command` (synchronous). Returns a
|
|
334
|
+
* {@link LockHandle} on success. Against a LIVE holder it WAITS by default
|
|
335
|
+
* (polling, blocking the thread) and acquires the moment the holder releases;
|
|
336
|
+
* pass `timeoutSec`/`wait` (or set `HQ_OP_LOCK_TIMEOUT`) to bound or disable the
|
|
337
|
+
* wait — on expiry it throws {@link OperationLockedError}. A stale lock (dead
|
|
338
|
+
* holder) is reclaimed immediately, never waited on.
|
|
339
|
+
*/
|
|
340
|
+
export function acquireOperationLock(hqRoot, command, opts = {}) {
|
|
341
|
+
if (process.env.HQ_DISABLE_OP_LOCK === "1")
|
|
342
|
+
return disabledHandle(hqRoot, command);
|
|
343
|
+
const { p, info, payload } = prepareLock(hqRoot, command);
|
|
344
|
+
const cfg = resolveWaitConfig(opts);
|
|
345
|
+
const deadline = cfg.timeoutMs === null ? null : Date.now() + cfg.timeoutMs;
|
|
346
|
+
let announced = false;
|
|
347
|
+
for (;;) {
|
|
348
|
+
const res = tryAcquireOnce(p, info, payload);
|
|
349
|
+
if ("handle" in res)
|
|
350
|
+
return res.handle;
|
|
351
|
+
// A live holder blocked us. Decide: refuse now, or wait and retry.
|
|
352
|
+
if (remainingMs(deadline) <= 0)
|
|
353
|
+
throw new OperationLockedError(res.busy, command);
|
|
354
|
+
if (!announced) {
|
|
355
|
+
announced = true;
|
|
356
|
+
cfg.onWaitStart(res.busy, command);
|
|
357
|
+
}
|
|
358
|
+
sleepSync(Math.min(cfg.pollMs, remainingMs(deadline)));
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Async counterpart to {@link acquireOperationLock}. Identical semantics, but
|
|
363
|
+
* the wait yields the event loop (via `setTimeout`) instead of blocking the
|
|
364
|
+
* thread — required for the async `sync` runner.
|
|
365
|
+
*/
|
|
366
|
+
export async function acquireOperationLockAsync(hqRoot, command, opts = {}) {
|
|
367
|
+
if (process.env.HQ_DISABLE_OP_LOCK === "1")
|
|
368
|
+
return disabledHandle(hqRoot, command);
|
|
369
|
+
const { p, info, payload } = prepareLock(hqRoot, command);
|
|
370
|
+
const cfg = resolveWaitConfig(opts);
|
|
371
|
+
const deadline = cfg.timeoutMs === null ? null : Date.now() + cfg.timeoutMs;
|
|
372
|
+
let announced = false;
|
|
373
|
+
for (;;) {
|
|
374
|
+
const res = tryAcquireOnce(p, info, payload);
|
|
375
|
+
if ("handle" in res)
|
|
376
|
+
return res.handle;
|
|
377
|
+
if (remainingMs(deadline) <= 0)
|
|
378
|
+
throw new OperationLockedError(res.busy, command);
|
|
379
|
+
if (!announced) {
|
|
380
|
+
announced = true;
|
|
381
|
+
cfg.onWaitStart(res.busy, command);
|
|
382
|
+
}
|
|
383
|
+
await sleepAsync(Math.min(cfg.pollMs, remainingMs(deadline)));
|
|
384
|
+
}
|
|
385
|
+
}
|
|
236
386
|
/** Run `fn` while holding the per-root lock for `command` (async). */
|
|
237
|
-
export async function withOperationLock(hqRoot, command, fn) {
|
|
238
|
-
const handle =
|
|
387
|
+
export async function withOperationLock(hqRoot, command, fn, opts = {}) {
|
|
388
|
+
const handle = await acquireOperationLockAsync(hqRoot, command, opts);
|
|
239
389
|
try {
|
|
240
390
|
return await fn();
|
|
241
391
|
}
|
|
@@ -244,8 +394,8 @@ export async function withOperationLock(hqRoot, command, fn) {
|
|
|
244
394
|
}
|
|
245
395
|
}
|
|
246
396
|
/** Run `fn` while holding the per-root lock for `command` (synchronous). */
|
|
247
|
-
export function withOperationLockSync(hqRoot, command, fn) {
|
|
248
|
-
const handle = acquireOperationLock(hqRoot, command);
|
|
397
|
+
export function withOperationLockSync(hqRoot, command, fn, opts = {}) {
|
|
398
|
+
const handle = acquireOperationLock(hqRoot, command, opts);
|
|
249
399
|
try {
|
|
250
400
|
return fn();
|
|
251
401
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operation-lock.js","sourceRoot":"","sources":["../src/operation-lock.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"operation-lock.js","sourceRoot":"","sources":["../src/operation-lock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+FG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,oFAAoF;AACpF,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAWxC,yEAAyE;AACzE,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAE3B;IACA;IAFlB,YACkB,MAAgB,EAChB,SAAiB;QAEjC,KAAK,CACH,sBAAsB,SAAS,qCAAqC;YAClE,+BAA+B,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,IAAI;YACrE,WAAW,MAAM,CAAC,SAAS,yCAAyC;YACpE,sBAAsB,CACzB,CAAC;QARc,WAAM,GAAN,MAAM,CAAU;QAChB,cAAS,GAAT,SAAS,CAAQ;QAQjC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAWD,4DAA4D;AAC5D,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAqCzC,sEAAsE;AACtE,SAAS,kBAAkB,CAAC,MAAgB,EAAE,SAAiB;IAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,gBAAgB,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,qBAAqB;QACrE,aAAa,SAAS,wDAAwD,CACjF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,IAAoB;IAC7C,uEAAuE;IACvE,0EAA0E;IAC1E,6BAA6B;IAC7B,MAAM,IAAI,GAAG,CAAC,GAAuB,EAAiB,EAAE;QACtD,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC;IAEF,IAAI,SAAwB,CAAC;IAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QAC9C,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClF,CAAC;IAED,0EAA0E;IAC1E,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;QAAE,SAAS,GAAG,CAAC,CAAC;IAEvC,MAAM,MAAM,GACV,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC;QAC5C,CAAC,CAAC,IAAI,CAAC,cAAc;QACrB,CAAC,CAAC,oBAAoB,CAAC;IAE3B,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,kBAAkB,EAAE,CAAC;AACpF,CAAC;AAED,gFAAgF;AAChF,SAAS,SAAS,CAAC,EAAU;IAC3B,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO;IACpB,4EAA4E;IAC5E,4EAA4E;IAC5E,4DAA4D;IAC5D,OAAO,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,iDAAiD;AACjD,SAAS,UAAU,CAAC,EAAU;IAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ;IACf,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;AACpE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/E,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAI,GAA6B,EAAE,IAAI,CAAC;QAClD,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,IAAI,CAAC,CAAC,6CAA6C;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAa,CAAC;QAClE,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnF,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,6EAA6E;AAC7E,6DAA6D;AAE7D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc,CAAC;AACxC,IAAI,cAAc,GAAG,KAAK,CAAC;AAE3B,SAAS,aAAa,CAAC,CAAS;IAC9B,0EAA0E;IAC1E,sEAAsE;IACtE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,cAAc;QAAE,OAAO;IAC3B,cAAc,GAAG,IAAI,CAAC;IAEtB,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACtB,KAAK,MAAM,CAAC,IAAI,SAAS;YAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QAC3D,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE;YACnB,KAAK,MAAM,CAAC,IAAI,SAAS;gBAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjD,yEAAyE;YACzE,yEAAyE;YACzE,iDAAiD;YACjD,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAAS,EAAE,IAAc;IAC3C,MAAM,MAAM,GAAe;QACzB,IAAI,EAAE,CAAC;QACP,IAAI;QACJ,OAAO;YACL,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,aAAa,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;KACF,CAAC;IACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,gBAAgB,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,gBAAgB,GAAG,EAAE,OAAO,KAAI,CAAC,EAAE,CAAC;AAE1C,gEAAgE;AAChE,SAAS,cAAc,CAAC,MAAc,EAAE,OAAe;IACrD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC7B,CAAC;IACF,OAAO,EAAE,GAAG,gBAAgB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACjD,CAAC;AAED,4DAA4D;AAC5D,SAAS,WAAW,CAAC,MAAc,EAAE,OAAe;IAClD,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC7B,CAAC;IACF,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CACrB,CAAS,EACT,IAAc,EACd,OAAe;IAEf,6EAA6E;IAC7E,4EAA4E;IAC5E,cAAc;IACd,MAAM,YAAY,GAAG,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,EAAU,CAAC;QACf,IAAI,CAAC;YACH,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,4BAA4B;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,EAAE,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YAC1B,CAAC;YACD,uEAAuE;YACvE,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;YACvE,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;IACzC,CAAC;IAED,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,IAAI,KAAK,CACb,0CAA0C,CAAC,UAAU,YAAY,WAAW,CAC7E,CAAC;AACJ,CAAC;AAED,gEAAgE;AAChE,SAAS,WAAW,CAAC,QAAuB;IAC1C,OAAO,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC9D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAc,EACd,OAAe,EACf,OAAuB,EAAE;IAEzB,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEnF,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC;IAC5E,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7C,IAAI,QAAQ,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC;QAEvC,mEAAmE;QACnE,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,IAAI,CAAC;YACjB,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,OAAe,EACf,OAAuB,EAAE;IAEzB,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEnF,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC;IAC5E,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7C,IAAI,QAAQ,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC;QAEvC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,IAAI,CAAC;YACjB,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;QACD,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,OAAe,EACf,EAAoB,EACpB,OAAuB,EAAE;IAEzB,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACtE,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,qBAAqB,CACnC,MAAc,EACd,OAAe,EACf,EAAW,EACX,OAAuB,EAAE;IAEzB,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}
|
|
@@ -6,7 +6,7 @@ import { spawnSync } from "child_process";
|
|
|
6
6
|
import * as fs from "fs";
|
|
7
7
|
import * as os from "os";
|
|
8
8
|
import * as path from "path";
|
|
9
|
-
import { acquireOperationLock, withOperationLockSync, lockPathFor, OperationLockedError, OPERATION_LOCKED_EXIT, } from "./operation-lock.js";
|
|
9
|
+
import { acquireOperationLock, acquireOperationLockAsync, withOperationLock, withOperationLockSync, lockPathFor, OperationLockedError, OPERATION_LOCKED_EXIT, DEFAULT_LOCK_POLL_MS, } from "./operation-lock.js";
|
|
10
10
|
/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
|
|
11
11
|
function deadPid() {
|
|
12
12
|
const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
|
|
@@ -33,6 +33,7 @@ describe("operation-lock", () => {
|
|
|
33
33
|
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
|
|
34
34
|
process.env.HQ_STATE_DIR = stateDir;
|
|
35
35
|
delete process.env.HQ_DISABLE_OP_LOCK;
|
|
36
|
+
delete process.env.HQ_OP_LOCK_TIMEOUT;
|
|
36
37
|
rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
|
|
37
38
|
rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
|
|
38
39
|
});
|
|
@@ -42,6 +43,7 @@ describe("operation-lock", () => {
|
|
|
42
43
|
fs.rmSync(rootB, { recursive: true, force: true });
|
|
43
44
|
delete process.env.HQ_STATE_DIR;
|
|
44
45
|
delete process.env.HQ_DISABLE_OP_LOCK;
|
|
46
|
+
delete process.env.HQ_OP_LOCK_TIMEOUT;
|
|
45
47
|
});
|
|
46
48
|
it("the lock path is under the state dir, keyed per canonical root", () => {
|
|
47
49
|
const a = lockPathFor(rootA);
|
|
@@ -60,14 +62,15 @@ describe("operation-lock", () => {
|
|
|
60
62
|
h.release();
|
|
61
63
|
expect(fs.existsSync(h.path)).toBe(false);
|
|
62
64
|
});
|
|
63
|
-
it("refuses
|
|
65
|
+
it("refuses immediately (wait:false) with the holder's command + pid when a LIVE process holds it", () => {
|
|
64
66
|
// Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
|
|
65
67
|
// is always alive and is never our own pid, so kill(1,0) reports alive and
|
|
66
|
-
// the same-process reclaim path does not apply.
|
|
68
|
+
// the same-process reclaim path does not apply. `wait:false` keeps the old
|
|
69
|
+
// refuse-immediately behavior (the default is now to WAIT).
|
|
67
70
|
writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
|
|
68
|
-
expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
|
|
71
|
+
expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(OperationLockedError);
|
|
69
72
|
try {
|
|
70
|
-
acquireOperationLock(rootA, "sync");
|
|
73
|
+
acquireOperationLock(rootA, "sync", { wait: false });
|
|
71
74
|
}
|
|
72
75
|
catch (e) {
|
|
73
76
|
const err = e;
|
|
@@ -77,11 +80,110 @@ describe("operation-lock", () => {
|
|
|
77
80
|
expect(err.message).toContain("pid 1");
|
|
78
81
|
}
|
|
79
82
|
});
|
|
80
|
-
it("
|
|
83
|
+
it("timeoutSec:0 refuses immediately (no wait) — equivalent to wait:false", () => {
|
|
84
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "sync" });
|
|
85
|
+
const start = Date.now();
|
|
86
|
+
expect(() => acquireOperationLock(rootA, "reindex", { timeoutSec: 0 })).toThrowError(OperationLockedError);
|
|
87
|
+
// Did not actually sleep.
|
|
88
|
+
expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
|
|
89
|
+
});
|
|
90
|
+
it("a bounded timeoutSec waits, then refuses with the old message + exit code", () => {
|
|
91
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
|
|
92
|
+
const start = Date.now();
|
|
93
|
+
let thrown;
|
|
94
|
+
try {
|
|
95
|
+
// 150ms bound, 40ms poll → waits ~150ms then gives up. Suppress the
|
|
96
|
+
// stderr status line with a no-op onWaitStart.
|
|
97
|
+
acquireOperationLock(rootA, "sync", {
|
|
98
|
+
timeoutSec: 0.15,
|
|
99
|
+
pollIntervalMs: 40,
|
|
100
|
+
onWaitStart: () => { },
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
catch (e) {
|
|
104
|
+
thrown = e;
|
|
105
|
+
}
|
|
106
|
+
const elapsed = Date.now() - start;
|
|
107
|
+
expect(thrown).toBeInstanceOf(OperationLockedError);
|
|
108
|
+
expect(thrown.message).toContain("rescue");
|
|
109
|
+
expect(OPERATION_LOCKED_EXIT).toBe(17);
|
|
110
|
+
// It actually waited (didn't refuse instantly) but didn't hang forever.
|
|
111
|
+
expect(elapsed).toBeGreaterThanOrEqual(120);
|
|
112
|
+
expect(elapsed).toBeLessThan(3000);
|
|
113
|
+
});
|
|
114
|
+
it("HQ_OP_LOCK_TIMEOUT env bounds the wait when no explicit option is given", () => {
|
|
115
|
+
process.env.HQ_OP_LOCK_TIMEOUT = "0"; // 0 → refuse immediately
|
|
116
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "sync" });
|
|
117
|
+
const start = Date.now();
|
|
118
|
+
expect(() => acquireOperationLock(rootA, "rescue", { onWaitStart: () => { } })).toThrowError(OperationLockedError);
|
|
119
|
+
expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
|
|
120
|
+
});
|
|
121
|
+
it("an explicit timeoutSec overrides the HQ_OP_LOCK_TIMEOUT env", () => {
|
|
122
|
+
process.env.HQ_OP_LOCK_TIMEOUT = "9999"; // would be a near-infinite wait
|
|
123
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "sync" });
|
|
124
|
+
// The explicit 0 wins → refuse immediately rather than honoring the env.
|
|
125
|
+
expect(() => acquireOperationLock(rootA, "rescue", { timeoutSec: 0 })).toThrowError(OperationLockedError);
|
|
126
|
+
});
|
|
127
|
+
it("onWaitStart fires exactly once, naming the holder, even across many polls", () => {
|
|
128
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
|
|
129
|
+
const calls = [];
|
|
130
|
+
expect(() => acquireOperationLock(rootA, "sync", {
|
|
131
|
+
timeoutSec: 0.16,
|
|
132
|
+
pollIntervalMs: 30, // ~5 polls within the window
|
|
133
|
+
onWaitStart: (holder, attempted) => calls.push({ cmd: holder.command, attempted }),
|
|
134
|
+
})).toThrowError(OperationLockedError);
|
|
135
|
+
expect(calls).toHaveLength(1);
|
|
136
|
+
expect(calls[0]).toEqual({ cmd: "rescue", attempted: "sync" });
|
|
137
|
+
});
|
|
138
|
+
it("a waiter acquires the lock the moment the holder releases (async poll path)", async () => {
|
|
139
|
+
const p = lockPathFor(rootA);
|
|
140
|
+
// A foreign LIVE holder (pid 1) initially owns the lock.
|
|
141
|
+
writeLock(p, { pid: 1, command: "rescue" });
|
|
142
|
+
// Simulate the holder finishing ~80ms in by removing its lock file.
|
|
143
|
+
const release = setTimeout(() => fs.rmSync(p, { force: true }), 80);
|
|
144
|
+
const start = Date.now();
|
|
145
|
+
const h = await acquireOperationLockAsync(rootA, "sync", {
|
|
146
|
+
pollIntervalMs: 20,
|
|
147
|
+
onWaitStart: () => { },
|
|
148
|
+
});
|
|
149
|
+
clearTimeout(release);
|
|
150
|
+
const elapsed = Date.now() - start;
|
|
151
|
+
// We waited for the release, then took it over.
|
|
152
|
+
expect(elapsed).toBeGreaterThanOrEqual(60);
|
|
153
|
+
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
154
|
+
expect(info.pid).toBe(process.pid);
|
|
155
|
+
expect(info.command).toBe("sync");
|
|
156
|
+
h.release();
|
|
157
|
+
});
|
|
158
|
+
it("multiple foreign holders in a row: each release lets the next waiter in (no FIFO guarantee)", async () => {
|
|
159
|
+
// The mutex is CROSS-PROCESS: it keys liveness on the holder's PID. Two
|
|
160
|
+
// waiters in the SAME process share a pid, so the same-process reclaim path
|
|
161
|
+
// would let them stomp each other — that scenario is unsupported by design.
|
|
162
|
+
// Here we model the real case: a sequence of FOREIGN holders (pid 1) that
|
|
163
|
+
// each release, with a single waiter acquiring the instant the lock frees.
|
|
164
|
+
// Order among multiple distinct-process waiters is whoever wins the next
|
|
165
|
+
// O_EXCL race after a free — best-effort, NOT FIFO (documented).
|
|
166
|
+
const p = lockPathFor(rootA);
|
|
167
|
+
writeLock(p, { pid: 1, command: "sync" });
|
|
168
|
+
// Free it shortly; the waiter should grab it right after.
|
|
169
|
+
setTimeout(() => fs.rmSync(p, { force: true }), 50);
|
|
170
|
+
const h = await acquireOperationLockAsync(rootA, "reindex", {
|
|
171
|
+
pollIntervalMs: 15,
|
|
172
|
+
onWaitStart: () => { },
|
|
173
|
+
});
|
|
174
|
+
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
175
|
+
expect(info.command).toBe("reindex");
|
|
176
|
+
expect(info.pid).toBe(process.pid);
|
|
177
|
+
h.release();
|
|
178
|
+
});
|
|
179
|
+
it("reclaims a stale lock whose holder PID is dead (takeover, never waits)", () => {
|
|
81
180
|
const stale = deadPid();
|
|
82
181
|
writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
|
|
83
|
-
|
|
182
|
+
const start = Date.now();
|
|
183
|
+
// The dead holder must not block us — even with an infinite default wait,
|
|
184
|
+
// takeover is immediate.
|
|
84
185
|
const h = acquireOperationLock(rootA, "rescue");
|
|
186
|
+
expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
|
|
85
187
|
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
86
188
|
expect(info.pid).toBe(process.pid); // we took it over
|
|
87
189
|
expect(info.command).toBe("rescue");
|
|
@@ -97,7 +199,7 @@ describe("operation-lock", () => {
|
|
|
97
199
|
});
|
|
98
200
|
it("different HQ roots are independent — both may hold concurrently", () => {
|
|
99
201
|
const a = acquireOperationLock(rootA, "sync");
|
|
100
|
-
const b = acquireOperationLock(rootB, "rescue"); // must NOT
|
|
202
|
+
const b = acquireOperationLock(rootB, "rescue"); // must NOT block
|
|
101
203
|
expect(fs.existsSync(a.path)).toBe(true);
|
|
102
204
|
expect(fs.existsSync(b.path)).toBe(true);
|
|
103
205
|
expect(a.path).not.toBe(b.path);
|
|
@@ -108,9 +210,10 @@ describe("operation-lock", () => {
|
|
|
108
210
|
// A live sync in ANOTHER process holds the root (pid 1 stands in for it).
|
|
109
211
|
const p = lockPathFor(rootA);
|
|
110
212
|
writeLock(p, { pid: 1, command: "sync" });
|
|
111
|
-
// Neither rescue nor reindex may acquire while that sync holds it
|
|
112
|
-
|
|
113
|
-
expect(() => acquireOperationLock(rootA, "
|
|
213
|
+
// Neither rescue nor reindex may acquire while that sync holds it
|
|
214
|
+
// (wait:false → assert the refusal without hanging on the new wait default).
|
|
215
|
+
expect(() => acquireOperationLock(rootA, "rescue", { wait: false })).toThrowError(OperationLockedError);
|
|
216
|
+
expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(OperationLockedError);
|
|
114
217
|
// Once that sync finishes (its lock is gone), the next command acquires.
|
|
115
218
|
fs.unlinkSync(p);
|
|
116
219
|
const h2 = acquireOperationLock(rootA, "reindex");
|
|
@@ -125,6 +228,14 @@ describe("operation-lock", () => {
|
|
|
125
228
|
})).toThrow("boom");
|
|
126
229
|
expect(fs.existsSync(p)).toBe(false); // released on the way out
|
|
127
230
|
});
|
|
231
|
+
it("withOperationLock (async) releases even when the body throws", async () => {
|
|
232
|
+
const p = lockPathFor(rootA);
|
|
233
|
+
await expect(withOperationLock(rootA, "sync", async () => {
|
|
234
|
+
expect(fs.existsSync(p)).toBe(true);
|
|
235
|
+
throw new Error("boom");
|
|
236
|
+
})).rejects.toThrow("boom");
|
|
237
|
+
expect(fs.existsSync(p)).toBe(false);
|
|
238
|
+
});
|
|
128
239
|
it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
|
|
129
240
|
process.env.HQ_DISABLE_OP_LOCK = "1";
|
|
130
241
|
// Even with a live holder on record, the escape hatch acquires without error.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operation-lock.test.js","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,WAAW,EACX,oBAAoB,EACpB,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AAE7B,yFAAyF;AACzF,SAAS,OAAO;IACd,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,IAAuB;IACnD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,CAAC;QACN,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QACpC,MAAM,EAAE,IAAI;QACZ,GAAG,IAAI;KACR,CAAC;IACF,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,KAAa,CAAC;IAClB,IAAI,KAAa,CAAC;IAElB,UAAU,CAAC,GAAG,EAAE;QACd,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC;QACpC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,yCAAyC;QAChE,4DAA4D;QAC5D,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC9E,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,2EAA2E;QAC3E,2EAA2E;QAC3E,gDAAgD;QAChD,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAyB,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC;QACxB,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/D,qCAAqC;QACrC,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,kBAAkB;QACnE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,0EAA0E;QAC1E,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,mEAAmE;QACnE,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACvF,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACxF,yEAAyE;QACzE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,EAAE,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,EAAE,CAAC,OAAO,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,EAAE,CACV,qBAAqB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YAC5D,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA0B;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,GAAG,CAAC;QACrC,8EAA8E;QAC9E,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACrD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,kBAAkB;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"operation-lock.test.js","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,iBAAiB,EACjB,qBAAqB,EACrB,WAAW,EACX,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,GAErB,MAAM,qBAAqB,CAAC;AAE7B,yFAAyF;AACzF,SAAS,OAAO;IACd,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,IAAuB;IACnD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,CAAC;QACN,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QACpC,MAAM,EAAE,IAAI;QACZ,GAAG,IAAI;KACR,CAAC;IACF,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,KAAa,CAAC;IAClB,IAAI,KAAa,CAAC;IAElB,UAAU,CAAC,GAAG,EAAE;QACd,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC;QACpC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,yCAAyC;QAChE,4DAA4D;QAC5D,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC9E,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+FAA+F,EAAE,GAAG,EAAE;QACvG,2EAA2E;QAC3E,2EAA2E;QAC3E,2EAA2E;QAC3E,4DAA4D;QAC5D,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAC7E,oBAAoB,CACrB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAyB,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAClF,oBAAoB,CACrB,CAAC;QACF,0BAA0B;QAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,oEAAoE;YACpE,+CAA+C;YAC/C,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE;gBAClC,UAAU,EAAE,IAAI;gBAChB,cAAc,EAAE,EAAE;gBAClB,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,CAAC,CAAC;QACb,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QACpD,MAAM,CAAE,MAA+B,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrE,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvC,wEAAwE;QACxE,MAAM,CAAC,OAAO,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,yBAAyB;QAC/D,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,EAAE,CACV,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC,CACjE,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,MAAM,CAAC,CAAC,gCAAgC;QACzE,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,yEAAyE;QACzE,MAAM,CAAC,GAAG,EAAE,CACV,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CACzD,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,GAA8C,EAAE,CAAC;QAC5D,MAAM,CAAC,GAAG,EAAE,CACV,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE;YAClC,UAAU,EAAE,IAAI;YAChB,cAAc,EAAE,EAAE,EAAE,6BAA6B;YACjD,WAAW,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC;SACnF,CAAC,CACH,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,yDAAyD;QACzD,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5C,oEAAoE;QACpE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE;YACvD,cAAc,EAAE,EAAE;YAClB,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC;SACtB,CAAC,CAAC;QACH,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACnC,gDAAgD;QAChD,MAAM,CAAC,OAAO,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6FAA6F,EAAE,KAAK,IAAI,EAAE;QAC3G,wEAAwE;QACxE,4EAA4E;QAC5E,4EAA4E;QAC5E,0EAA0E;QAC1E,2EAA2E;QAC3E,yEAAyE;QACzE,iEAAiE;QACjE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,0DAA0D;QAC1D,UAAU,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,SAAS,EAAE;YAC1D,cAAc,EAAE,EAAE;YAClB,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC;SACtB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAChF,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC;QACxB,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,0EAA0E;QAC1E,yBAAyB;QACzB,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,iBAAiB;QAClE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,0EAA0E;QAC1E,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,kEAAkE;QAClE,6EAA6E;QAC7E,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAC/E,oBAAoB,CACrB,CAAC;QACF,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAChF,oBAAoB,CACrB,CAAC;QACF,yEAAyE;QACzE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,EAAE,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,EAAE,CAAC,OAAO,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,EAAE,CACV,qBAAqB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YAC5D,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA0B;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,MAAM,CACV,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,GAAG,CAAC;QACrC,8EAA8E;QAC9E,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACrD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,kBAAkB;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|