@indigoai-us/hq-cloud 6.2.6 → 6.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/bin/sync-runner.d.ts +8 -7
  2. package/dist/bin/sync-runner.d.ts.map +1 -1
  3. package/dist/bin/sync-runner.js +51 -9
  4. package/dist/bin/sync-runner.js.map +1 -1
  5. package/dist/bin/sync-runner.test.js +74 -2
  6. package/dist/bin/sync-runner.test.js.map +1 -1
  7. package/dist/cli/reindex.d.ts +8 -0
  8. package/dist/cli/reindex.d.ts.map +1 -1
  9. package/dist/cli/reindex.js +222 -198
  10. package/dist/cli/reindex.js.map +1 -1
  11. package/dist/cli/reindex.test.js +35 -0
  12. package/dist/cli/reindex.test.js.map +1 -1
  13. package/dist/cli/rescue.d.ts.map +1 -1
  14. package/dist/cli/rescue.js +39 -16
  15. package/dist/cli/rescue.js.map +1 -1
  16. package/dist/cli/rescue.reindex.test.js +15 -2
  17. package/dist/cli/rescue.reindex.test.js.map +1 -1
  18. package/dist/cli/sync.d.ts.map +1 -1
  19. package/dist/cli/sync.js +3 -1
  20. package/dist/cli/sync.js.map +1 -1
  21. package/dist/cli/sync.test.js +2 -1
  22. package/dist/cli/sync.test.js.map +1 -1
  23. package/dist/operation-lock.d.ts +100 -0
  24. package/dist/operation-lock.d.ts.map +1 -0
  25. package/dist/operation-lock.js +256 -0
  26. package/dist/operation-lock.js.map +1 -0
  27. package/dist/operation-lock.test.d.ts +5 -0
  28. package/dist/operation-lock.test.d.ts.map +1 -0
  29. package/dist/operation-lock.test.js +140 -0
  30. package/dist/operation-lock.test.js.map +1 -0
  31. package/package.json +1 -1
  32. package/src/bin/sync-runner.test.ts +91 -2
  33. package/src/bin/sync-runner.ts +52 -9
  34. package/src/cli/reindex.test.ts +45 -0
  35. package/src/cli/reindex.ts +36 -0
  36. package/src/cli/rescue.reindex.test.ts +17 -2
  37. package/src/cli/rescue.ts +40 -15
  38. package/src/cli/sync.test.ts +2 -1
  39. package/src/cli/sync.ts +3 -1
  40. package/src/operation-lock.test.ts +162 -0
  41. package/src/operation-lock.ts +293 -0
@@ -0,0 +1,256 @@
1
+ /**
2
+ * Per-HQ-root mutual exclusion for the long-running operations
3
+ * (`sync`, `rescue`, `reindex`).
4
+ *
5
+ * Contract:
6
+ * - At most ONE of sync / rescue / reindex runs at a time **per HQ root**.
7
+ * The lock is shared across all three (keyed only by the root, not the
8
+ * command), so e.g. a rescue refuses while a sync holds it. Different HQ
9
+ * roots are fully independent — they hash to different lock files.
10
+ * - The push watcher / watch+event-push runner is EXEMPT: it never calls in
11
+ * here, so it neither takes the lock nor is blocked by it (its targeted
12
+ * in-process push passes are likewise lock-free).
13
+ *
14
+ * ## Where the lock lives — and why
15
+ *
16
+ * `<stateDir>/locks/operation-<hash(canonicalRoot)>.lock`, where
17
+ * `stateDir = $HQ_STATE_DIR || ~/.hq`. This is deliberately NOT inside the HQ
18
+ * root:
19
+ * - It must never round-trip to the cloud. A lock is machine-local, per-run
20
+ * state; syncing it to S3 (and thence to other machines/roots) would be a
21
+ * correctness bug. `~/.hq` is the established machine-local state dir
22
+ * (journals already live there) and is never synced.
23
+ * - `rescue` repairs a possibly-broken HQ root; a lock that depends on the
24
+ * root being healthy is exactly backwards. `~/.hq` is independent of the
25
+ * root's health.
26
+ * - Keying the filename by a hash of the *canonical* root path makes the
27
+ * lock per-root and prevents leakage across roots, while keeping the path
28
+ * short and filesystem-safe.
29
+ *
30
+ * ## Atomicity, liveness, takeover
31
+ *
32
+ * - Acquisition uses `open(…, "wx")` (O_CREAT | O_EXCL) — an atomic
33
+ * create-if-absent. Exactly one racer can create the file; the loser sees
34
+ * EEXIST and re-evaluates.
35
+ * - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
36
+ * EEXIST we test the recorded PID with `process.kill(pid, 0)`:
37
+ * * ESRCH → the holder is gone (crashed / killed -9 / stale file) →
38
+ * reclaim the lock.
39
+ * * EPERM → the PID exists but is owned by another user → treat as ALIVE
40
+ * (conservative: refuse rather than risk two concurrent ops).
41
+ * * success → alive → refuse fast with {@link OperationLockedError}
42
+ * naming the holding command + PID.
43
+ * - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
44
+ * the original holder crashed and the OS later handed its PID to an
45
+ * unrelated process, we conservatively read that as "still held" and
46
+ * refuse. We accept that false-busy over the far worse false-free, and
47
+ * record `startedAt`/`command` so an operator can diagnose a wedged lock.
48
+ *
49
+ * ## Release
50
+ *
51
+ * - Normal exit: the `with*` wrappers release in a `finally`.
52
+ * - Signals (SIGINT/SIGTERM): a one-time handler releases every held lock,
53
+ * then re-raises the default disposition so exit status is unchanged.
54
+ * - Hard crash (SIGKILL / power loss): nothing runs, but the stale-PID
55
+ * takeover above reclaims the lock on the next attempt.
56
+ * - `process.on("exit")`: a final best-effort synchronous unlink.
57
+ *
58
+ * ## Escape hatch
59
+ *
60
+ * `HQ_DISABLE_OP_LOCK=1` makes acquisition a no-op (returns a handle whose
61
+ * release does nothing). For emergencies and for callers that manage
62
+ * exclusion themselves; documented, off by default.
63
+ */
64
+ import * as crypto from "crypto";
65
+ import * as fs from "fs";
66
+ import * as os from "os";
67
+ import * as path from "path";
68
+ /** Process exit code used when an operation is refused because the lock is held. */
69
+ export const OPERATION_LOCKED_EXIT = 17;
70
+ /** Thrown by `acquireOperationLock` when a LIVE holder owns the lock. */
71
+ export class OperationLockedError extends Error {
72
+ holder;
73
+ attempted;
74
+ constructor(holder, attempted) {
75
+ super(`Refusing to start "${attempted}": another HQ operation is already ` +
76
+ `running for this HQ root — "${holder.command}" (pid ${holder.pid}, ` +
77
+ `started ${holder.startedAt}). Wait for it to finish, or stop that ` +
78
+ `process, then retry.`);
79
+ this.holder = holder;
80
+ this.attempted = attempted;
81
+ this.name = "OperationLockedError";
82
+ }
83
+ }
84
+ function stateDir() {
85
+ return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
86
+ }
87
+ /** Absolute lock path for a given HQ root. Exported for tests. */
88
+ export function lockPathFor(hqRoot) {
89
+ const canon = path.resolve(hqRoot);
90
+ const key = crypto.createHash("sha1").update(canon).digest("hex").slice(0, 16);
91
+ return path.join(stateDir(), "locks", `operation-${key}.lock`);
92
+ }
93
+ /**
94
+ * Is `pid` a live process? `kill(pid, 0)` sends no signal; it only probes.
95
+ * ESRCH → no such process (dead/stale). EPERM → exists but not ours → ALIVE
96
+ * (conservative). Anything else → assume alive rather than risk a double-run.
97
+ */
98
+ function pidAlive(pid) {
99
+ if (!Number.isInteger(pid) || pid <= 0)
100
+ return false;
101
+ try {
102
+ process.kill(pid, 0);
103
+ return true;
104
+ }
105
+ catch (err) {
106
+ const code = err?.code;
107
+ if (code === "ESRCH")
108
+ return false;
109
+ return true; // EPERM (exists) or unknown → treat as alive
110
+ }
111
+ }
112
+ function readLockInfo(p) {
113
+ try {
114
+ const parsed = JSON.parse(fs.readFileSync(p, "utf8"));
115
+ if (parsed && typeof parsed.pid === "number" && typeof parsed.command === "string") {
116
+ return parsed;
117
+ }
118
+ return null;
119
+ }
120
+ catch {
121
+ return null;
122
+ }
123
+ }
124
+ // ── Process-wide release plumbing ──────────────────────────────────────────
125
+ // Track every lock this process currently holds so the signal/exit hooks can
126
+ // release all of them. The hooks are installed exactly once.
127
+ const heldLocks = new Set();
128
+ let hooksInstalled = false;
129
+ function unlinkIfOwned(p) {
130
+ // Only remove a lock whose recorded pid is THIS process — never clobber a
131
+ // lock another process took over after a (hypothetical) reclaim race.
132
+ const info = readLockInfo(p);
133
+ if (info && info.pid === process.pid) {
134
+ try {
135
+ fs.unlinkSync(p);
136
+ }
137
+ catch {
138
+ /* already gone — fine */
139
+ }
140
+ }
141
+ }
142
+ function installHooksOnce() {
143
+ if (hooksInstalled)
144
+ return;
145
+ hooksInstalled = true;
146
+ process.on("exit", () => {
147
+ for (const h of heldLocks)
148
+ unlinkIfOwned(h.path);
149
+ });
150
+ for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
151
+ process.on(sig, () => {
152
+ for (const h of heldLocks)
153
+ unlinkIfOwned(h.path);
154
+ // Re-raise with the default disposition so the exit status is the normal
155
+ // signal status (and a second Ctrl-C still works). Removing our listener
156
+ // first avoids recursing back into this handler.
157
+ process.removeAllListeners(sig);
158
+ process.kill(process.pid, sig);
159
+ });
160
+ }
161
+ }
162
+ function makeHandle(p, info) {
163
+ const handle = {
164
+ path: p,
165
+ info,
166
+ release() {
167
+ heldLocks.delete(handle);
168
+ unlinkIfOwned(p);
169
+ },
170
+ };
171
+ heldLocks.add(handle);
172
+ installHooksOnce();
173
+ return handle;
174
+ }
175
+ const NOOP_HANDLE_BASE = { release() { } };
176
+ /**
177
+ * Acquire the per-root operation lock for `command`. Returns a {@link LockHandle}
178
+ * on success; throws {@link OperationLockedError} when a live holder owns it.
179
+ * Reclaims a stale lock (dead holder) transparently.
180
+ */
181
+ export function acquireOperationLock(hqRoot, command) {
182
+ if (process.env.HQ_DISABLE_OP_LOCK === "1") {
183
+ const info = {
184
+ pid: process.pid,
185
+ command,
186
+ startedAt: new Date().toISOString(),
187
+ hqRoot: path.resolve(hqRoot),
188
+ };
189
+ return { ...NOOP_HANDLE_BASE, path: "", info };
190
+ }
191
+ const p = lockPathFor(hqRoot);
192
+ fs.mkdirSync(path.dirname(p), { recursive: true });
193
+ const info = {
194
+ pid: process.pid,
195
+ command,
196
+ startedAt: new Date().toISOString(),
197
+ hqRoot: path.resolve(hqRoot),
198
+ };
199
+ const payload = JSON.stringify(info, null, 2);
200
+ // Bounded retry: each iteration is one atomic create attempt. EEXIST against
201
+ // a stale holder reclaims and retries; EEXIST against a live holder refuses.
202
+ const MAX_ATTEMPTS = 5;
203
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
204
+ let fd;
205
+ try {
206
+ fd = fs.openSync(p, "wx"); // O_CREAT | O_EXCL — atomic
207
+ }
208
+ catch (err) {
209
+ if (err?.code !== "EEXIST")
210
+ throw err;
211
+ const holder = readLockInfo(p);
212
+ if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
213
+ throw new OperationLockedError(holder, command);
214
+ }
215
+ // Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
216
+ try {
217
+ fs.unlinkSync(p);
218
+ }
219
+ catch {
220
+ /* someone else reclaimed it first; the next openSync re-evaluates */
221
+ }
222
+ continue;
223
+ }
224
+ try {
225
+ fs.writeSync(fd, payload);
226
+ }
227
+ finally {
228
+ fs.closeSync(fd);
229
+ }
230
+ return makeHandle(p, info);
231
+ }
232
+ // Pathological churn (another process reclaiming in lockstep). Surface it
233
+ // rather than spin forever.
234
+ throw new Error(`Could not acquire HQ operation lock at ${p} after ${MAX_ATTEMPTS} attempts`);
235
+ }
236
+ /** Run `fn` while holding the per-root lock for `command` (async). */
237
+ export async function withOperationLock(hqRoot, command, fn) {
238
+ const handle = acquireOperationLock(hqRoot, command);
239
+ try {
240
+ return await fn();
241
+ }
242
+ finally {
243
+ handle.release();
244
+ }
245
+ }
246
+ /** Run `fn` while holding the per-root lock for `command` (synchronous). */
247
+ export function withOperationLockSync(hqRoot, command, fn) {
248
+ const handle = acquireOperationLock(hqRoot, command);
249
+ try {
250
+ return fn();
251
+ }
252
+ finally {
253
+ handle.release();
254
+ }
255
+ }
256
+ //# sourceMappingURL=operation-lock.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"operation-lock.js","sourceRoot":"","sources":["../src/operation-lock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,oFAAoF;AACpF,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAWxC,yEAAyE;AACzE,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAE3B;IACA;IAFlB,YACkB,MAAgB,EAChB,SAAiB;QAEjC,KAAK,CACH,sBAAsB,SAAS,qCAAqC;YAClE,+BAA+B,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,IAAI;YACrE,WAAW,MAAM,CAAC,SAAS,yCAAyC;YACpE,sBAAsB,CACzB,CAAC;QARc,WAAM,GAAN,MAAM,CAAU;QAChB,cAAS,GAAT,SAAS,CAAQ;QAQjC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAWD,SAAS,QAAQ;IACf,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;AACpE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/E,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAI,GAA6B,EAAE,IAAI,CAAC;QAClD,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,IAAI,CAAC,CAAC,6CAA6C;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAa,CAAC;QAClE,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnF,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,6EAA6E;AAC7E,6DAA6D;AAE7D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc,CAAC;AACxC,IAAI,cAAc,GAAG,KAAK,CAAC;AAE3B,SAAS,aAAa,CAAC,CAAS;IAC9B,0EAA0E;IAC1E,sEAAsE;IACtE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,cAAc;QAAE,OAAO;IAC3B,cAAc,GAAG,IAAI,CAAC;IAEtB,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACtB,KAAK,MAAM,CAAC,IAAI,SAAS;YAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QAC3D,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE;YACnB,KAAK,MAAM,CAAC,IAAI,SAAS;gBAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjD,yEAAyE;YACzE,yEAAyE;YACzE,iDAAiD;YACjD,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAAS,EAAE,IAAc;IAC3C,MAAM,MAAM,GAAe;QACzB,IAAI,EAAE,CAAC;QACP,IAAI;QACJ,OAAO;YACL,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,aAAa,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;KACF,CAAC;IACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,gBAAgB,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,gBAAgB,GAAG,EAAE,OAAO,KAAI,CAAC,EAAE,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,OAAe;IAClE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAa;YACrB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO;YACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;SAC7B,CAAC;QACF,OAAO,EAAE,GAAG,gBAAgB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IACjD,CAAC;IAED,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC7B,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAE9C,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,YAAY,GAAG,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,EAAU,CAAC;QACf,IAAI,CAAC;YACH,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,4BAA4B;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,EAAE,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;YACD,uEAAuE;YACvE,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;YACvE,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,IAAI,KAAK,CACb,0CAA0C,CAAC,UAAU,YAAY,WAAW,CAC7E,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,OAAe,EACf,EAAoB;IAEpB,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,qBAAqB,CACnC,MAAc,EACd,OAAe,EACf,EAAW;IAEX,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Unit tests for the per-HQ-root operation mutex.
3
+ */
4
+ export {};
5
+ //# sourceMappingURL=operation-lock.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"operation-lock.test.d.ts","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}
@@ -0,0 +1,140 @@
1
+ /**
2
+ * Unit tests for the per-HQ-root operation mutex.
3
+ */
4
+ import { describe, it, expect, beforeEach, afterEach } from "vitest";
5
+ import { spawnSync } from "child_process";
6
+ import * as fs from "fs";
7
+ import * as os from "os";
8
+ import * as path from "path";
9
+ import { acquireOperationLock, withOperationLockSync, lockPathFor, OperationLockedError, OPERATION_LOCKED_EXIT, } from "./operation-lock.js";
10
+ /** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
11
+ function deadPid() {
12
+ const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
13
+ if (!r.pid)
14
+ throw new Error("could not spawn to obtain a dead pid");
15
+ return r.pid;
16
+ }
17
+ function writeLock(p, info) {
18
+ fs.mkdirSync(path.dirname(p), { recursive: true });
19
+ const full = {
20
+ pid: 1,
21
+ command: "sync",
22
+ startedAt: new Date(0).toISOString(),
23
+ hqRoot: "/x",
24
+ ...info,
25
+ };
26
+ fs.writeFileSync(p, JSON.stringify(full));
27
+ }
28
+ describe("operation-lock", () => {
29
+ let stateDir;
30
+ let rootA;
31
+ let rootB;
32
+ beforeEach(() => {
33
+ stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
34
+ process.env.HQ_STATE_DIR = stateDir;
35
+ delete process.env.HQ_DISABLE_OP_LOCK;
36
+ rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
37
+ rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
38
+ });
39
+ afterEach(() => {
40
+ fs.rmSync(stateDir, { recursive: true, force: true });
41
+ fs.rmSync(rootA, { recursive: true, force: true });
42
+ fs.rmSync(rootB, { recursive: true, force: true });
43
+ delete process.env.HQ_STATE_DIR;
44
+ delete process.env.HQ_DISABLE_OP_LOCK;
45
+ });
46
+ it("the lock path is under the state dir, keyed per canonical root", () => {
47
+ const a = lockPathFor(rootA);
48
+ const b = lockPathFor(rootB);
49
+ expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
50
+ expect(a).not.toBe(b); // different roots → different lock files
51
+ // canonical: a trailing-slash variant maps to the same lock
52
+ expect(lockPathFor(rootA + path.sep)).toBe(a);
53
+ });
54
+ it("acquires, writes holder info, and releases (file gone after release)", () => {
55
+ const h = acquireOperationLock(rootA, "sync");
56
+ expect(fs.existsSync(h.path)).toBe(true);
57
+ const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
58
+ expect(info.pid).toBe(process.pid);
59
+ expect(info.command).toBe("sync");
60
+ h.release();
61
+ expect(fs.existsSync(h.path)).toBe(false);
62
+ });
63
+ it("refuses fast with the holder's command + pid when a LIVE process holds it", () => {
64
+ // Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
65
+ // is always alive and is never our own pid, so kill(1,0) reports alive and
66
+ // the same-process reclaim path does not apply.
67
+ writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
68
+ expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
69
+ try {
70
+ acquireOperationLock(rootA, "sync");
71
+ }
72
+ catch (e) {
73
+ const err = e;
74
+ expect(err.holder.command).toBe("rescue");
75
+ expect(err.holder.pid).toBe(1);
76
+ expect(err.message).toContain("rescue");
77
+ expect(err.message).toContain("pid 1");
78
+ }
79
+ });
80
+ it("reclaims a stale lock whose holder PID is dead (takeover)", () => {
81
+ const stale = deadPid();
82
+ writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
83
+ // The dead holder must not block us.
84
+ const h = acquireOperationLock(rootA, "rescue");
85
+ const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
86
+ expect(info.pid).toBe(process.pid); // we took it over
87
+ expect(info.command).toBe("rescue");
88
+ h.release();
89
+ });
90
+ it("reclaims a torn/unreadable lock file", () => {
91
+ const p = lockPathFor(rootA);
92
+ fs.mkdirSync(path.dirname(p), { recursive: true });
93
+ fs.writeFileSync(p, "{ this is not valid json");
94
+ const h = acquireOperationLock(rootA, "reindex");
95
+ expect(fs.existsSync(h.path)).toBe(true);
96
+ h.release();
97
+ });
98
+ it("different HQ roots are independent — both may hold concurrently", () => {
99
+ const a = acquireOperationLock(rootA, "sync");
100
+ const b = acquireOperationLock(rootB, "rescue"); // must NOT refuse
101
+ expect(fs.existsSync(a.path)).toBe(true);
102
+ expect(fs.existsSync(b.path)).toBe(true);
103
+ expect(a.path).not.toBe(b.path);
104
+ a.release();
105
+ b.release();
106
+ });
107
+ it("the same root is mutually exclusive across different commands", () => {
108
+ // A live sync in ANOTHER process holds the root (pid 1 stands in for it).
109
+ const p = lockPathFor(rootA);
110
+ writeLock(p, { pid: 1, command: "sync" });
111
+ // Neither rescue nor reindex may acquire while that sync holds it.
112
+ expect(() => acquireOperationLock(rootA, "rescue")).toThrowError(OperationLockedError);
113
+ expect(() => acquireOperationLock(rootA, "reindex")).toThrowError(OperationLockedError);
114
+ // Once that sync finishes (its lock is gone), the next command acquires.
115
+ fs.unlinkSync(p);
116
+ const h2 = acquireOperationLock(rootA, "reindex");
117
+ expect(fs.existsSync(h2.path)).toBe(true);
118
+ h2.release();
119
+ });
120
+ it("withOperationLockSync releases even when the body throws", () => {
121
+ const p = lockPathFor(rootA);
122
+ expect(() => withOperationLockSync(rootA, "reindex", () => {
123
+ expect(fs.existsSync(p)).toBe(true); // held during the body
124
+ throw new Error("boom");
125
+ })).toThrow("boom");
126
+ expect(fs.existsSync(p)).toBe(false); // released on the way out
127
+ });
128
+ it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
129
+ process.env.HQ_DISABLE_OP_LOCK = "1";
130
+ // Even with a live holder on record, the escape hatch acquires without error.
131
+ writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
132
+ const h = acquireOperationLock(rootA, "rescue");
133
+ expect(h.path).toBe(""); // no real lock file written
134
+ h.release(); // no-op, no throw
135
+ });
136
+ it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
137
+ expect(OPERATION_LOCKED_EXIT).toBe(17);
138
+ });
139
+ });
140
+ //# sourceMappingURL=operation-lock.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"operation-lock.test.js","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,WAAW,EACX,oBAAoB,EACpB,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AAE7B,yFAAyF;AACzF,SAAS,OAAO;IACd,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,IAAuB;IACnD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,CAAC;QACN,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QACpC,MAAM,EAAE,IAAI;QACZ,GAAG,IAAI;KACR,CAAC;IACF,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,KAAa,CAAC;IAClB,IAAI,KAAa,CAAC;IAElB,UAAU,CAAC,GAAG,EAAE;QACd,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC;QACpC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,yCAAyC;QAChE,4DAA4D;QAC5D,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC9E,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,2EAA2E;QAC3E,2EAA2E;QAC3E,gDAAgD;QAChD,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAyB,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC;QACxB,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/D,qCAAqC;QACrC,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,kBAAkB;QACnE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,0EAA0E;QAC1E,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,mEAAmE;QACnE,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACvF,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACxF,yEAAyE;QACzE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,EAAE,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,EAAE,CAAC,OAAO,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,EAAE,CACV,qBAAqB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YAC5D,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA0B;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,GAAG,CAAC;QACrC,8EAA8E;QAC9E,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACrD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,kBAAkB;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cloud",
3
- "version": "6.2.6",
3
+ "version": "6.3.0",
4
4
  "description": "HQ by Indigo cloud sync engine \u2014 bidirectional S3 sync for mobile access",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -30,6 +30,7 @@ import type {
30
30
  } from "./sync-runner.js";
31
31
  import { FakeClock } from "../watcher.js";
32
32
  import { PERSONAL_VAULT_JOURNAL_SLUG } from "../journal.js";
33
+ import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
33
34
  import type { SyncResult, SyncOptions } from "../cli/sync.js";
34
35
  import type { ShareResult, ShareOptions } from "../cli/share.js";
35
36
  import type {
@@ -3176,13 +3177,25 @@ describe("resolvePresignTransport", () => {
3176
3177
  expect(resolvePresignTransport("ME@GetIndigo.AI", undefined)).toBe(true);
3177
3178
  });
3178
3179
 
3179
- it("OFF for non-getindigo emails, no override", () => {
3180
+ it("ON for batch-2 rollout domains (gmail.com, vyg.ai, amass.com), no override", () => {
3181
+ expect(resolvePresignTransport("someone@gmail.com", undefined)).toBe(true);
3182
+ expect(resolvePresignTransport("Someone@GMAIL.com", undefined)).toBe(true);
3183
+ expect(resolvePresignTransport("shahzaib@vyg.ai", undefined)).toBe(true);
3184
+ expect(resolvePresignTransport("ops@amass.com", undefined)).toBe(true);
3185
+ });
3186
+
3187
+ it("OFF for unenrolled emails, no override", () => {
3180
3188
  expect(resolvePresignTransport("me@example.com", undefined)).toBe(false);
3181
3189
  expect(resolvePresignTransport(undefined, undefined)).toBe(false);
3182
- // A lookalike domain must not match the endsWith check loosely.
3190
+ expect(resolvePresignTransport("no-at-sign", undefined)).toBe(false);
3191
+ // Exact-domain matching — lookalikes and suffix tricks must not match.
3183
3192
  expect(resolvePresignTransport("me@notgetindigo.ai.evil.com", undefined)).toBe(
3184
3193
  false,
3185
3194
  );
3195
+ expect(resolvePresignTransport("me@gmail.com.evil.com", undefined)).toBe(false);
3196
+ expect(resolvePresignTransport("me@evil-gmail.com", undefined)).toBe(false);
3197
+ // ridge.com is deliberately NOT in batch 2.
3198
+ expect(resolvePresignTransport("juan@ridge.com", undefined)).toBe(false);
3186
3199
  });
3187
3200
 
3188
3201
  it.each(["1", "true", "yes", "on", "ON", " True "])(
@@ -3554,3 +3567,79 @@ describe("readPinnedPrefixes", () => {
3554
3567
  expect(readPinnedPrefixes(root, "acme")).toEqual([]);
3555
3568
  });
3556
3569
  });
3570
+
3571
+ // ---------------------------------------------------------------------------
3572
+ // Operation lock — one-shot sync takes it; the watch runner is exempt.
3573
+ // ---------------------------------------------------------------------------
3574
+ describe("runRunnerWithLoop — operation lock", () => {
3575
+ const HQ = "/tmp/hq-oplock";
3576
+
3577
+ /** Write a live-holder lock (pid 1) for the HQ root into the test state dir. */
3578
+ function writeLiveHolder(command: string): string {
3579
+ const p = lockPathFor(HQ);
3580
+ fs.mkdirSync(path.dirname(p), { recursive: true });
3581
+ fs.writeFileSync(
3582
+ p,
3583
+ JSON.stringify({ pid: 1, command, startedAt: new Date(0).toISOString(), hqRoot: HQ }),
3584
+ );
3585
+ return p;
3586
+ }
3587
+
3588
+ it("one-shot sync refuses fast (exit 17) when another op holds the root", async () => {
3589
+ const lp = writeLiveHolder("rescue");
3590
+ const errs: string[] = [];
3591
+ const spy = vi
3592
+ .spyOn(process.stderr, "write")
3593
+ .mockImplementation((chunk: string | Uint8Array) => {
3594
+ errs.push(String(chunk));
3595
+ return true;
3596
+ });
3597
+
3598
+ // No --watch → one-shot. Refusal short-circuits BEFORE runRunner (so no
3599
+ // network / auth is touched).
3600
+ const code = await runRunnerWithLoop(["--companies", "--hq-root", HQ]);
3601
+
3602
+ spy.mockRestore();
3603
+ expect(code).toBe(OPERATION_LOCKED_EXIT);
3604
+ expect(errs.join("")).toContain("rescue"); // names the holder
3605
+ // The holder's lock is left intact — we refused, we didn't take it over.
3606
+ const held = JSON.parse(fs.readFileSync(lp, "utf8"));
3607
+ expect(held.pid).toBe(1);
3608
+ expect(held.command).toBe("rescue");
3609
+ });
3610
+
3611
+ it("the watch runner is EXEMPT — runs despite a held lock and never takes it", async () => {
3612
+ const lp = writeLiveHolder("sync");
3613
+ const watcher = makeWatcherStub();
3614
+ let triggerShutdown = () => {};
3615
+ const runPass = vi.fn().mockResolvedValue(0);
3616
+
3617
+ const loop = runRunnerWithLoop(
3618
+ ["--companies", "--watch", "--event-push", "--hq-root", HQ],
3619
+ {
3620
+ runPass,
3621
+ clock: new FakeClock(),
3622
+ createWatcher: () => watcher,
3623
+ sleep: () => new Promise<void>(() => {}),
3624
+ onShutdownSignal: (handler) => {
3625
+ triggerShutdown = handler;
3626
+ return () => {};
3627
+ },
3628
+ },
3629
+ );
3630
+
3631
+ await Promise.resolve();
3632
+ await Promise.resolve();
3633
+ // It started and ran a pass even though the lock is held → not blocked.
3634
+ expect(watcher.started).toBe(true);
3635
+ expect(runPass).toHaveBeenCalled();
3636
+
3637
+ triggerShutdown();
3638
+ await loop;
3639
+
3640
+ // The pre-existing holder lock is untouched → the watcher never took it.
3641
+ const held = JSON.parse(fs.readFileSync(lp, "utf8"));
3642
+ expect(held.pid).toBe(1);
3643
+ expect(held.command).toBe("sync");
3644
+ });
3645
+ });
@@ -100,6 +100,11 @@ import { collectAndSendTelemetry } from "../telemetry.js";
100
100
  import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
101
101
  import { reindexAfterSync } from "../qmd-reindex.js";
102
102
  import { pruneConflictIndex } from "../lib/conflict-index.js";
103
+ import {
104
+ withOperationLock,
105
+ OperationLockedError,
106
+ OPERATION_LOCKED_EXIT,
107
+ } from "../operation-lock.js";
103
108
  import { describeError } from "../lib/describe-error.js";
104
109
  import { getOrCreateMachineId } from "../lib/machine-id.js";
105
110
  import {
@@ -210,16 +215,35 @@ export function resolveSkipPersonal(flag: boolean): boolean {
210
215
  return env === "1" || env === "true" || env === "yes";
211
216
  }
212
217
 
218
+ /**
219
+ * Email domains enrolled in the presigned-URL transport rollout.
220
+ *
221
+ * Staged by the operator (2026-06-10) in descending user count: getindigo.ai
222
+ * was the pilot; gmail.com / vyg.ai / amass.com are batch 2. The presign
223
+ * transport also routes every upload through the vault API's
224
+ * `validateObjectKey` (INVALID_KEY_BACKSLASH et al.), so enrolling a domain
225
+ * upgrades its server-side input validation versus the STS-direct path.
226
+ * Matching is on the EXACT domain (the part after the last `@`), not a
227
+ * suffix — `evil-gmail.com` and `gmail.com.evil.com` never match.
228
+ */
229
+ const PRESIGN_ROLLOUT_DOMAINS: ReadonlySet<string> = new Set([
230
+ "getindigo.ai",
231
+ "gmail.com",
232
+ "vyg.ai",
233
+ "amass.com",
234
+ ]);
235
+
213
236
  /**
214
237
  * Decide whether this session uses the presigned-URL transport.
215
238
  *
216
- * Rollout gate: ON for accounts whose verified email is `@getindigo.ai`.
217
- * `HQ_SYNC_PRESIGN_TRANSPORT` overrides the email check in both directions
218
- * (`1`/`true`/`yes`/`on` → force on, `0`/`false`/`no`/`off` → force off) so
219
- * the transport can be exercised by non-getindigo testers or rolled back for
220
- * getindigo accounts without a redeploy. An unset/blank override falls through
221
- * to the email check; an unrecognized override value is ignored (email check
222
- * wins) rather than silently forcing a state.
239
+ * Rollout gate: ON for accounts whose verified email domain is enrolled in
240
+ * `PRESIGN_ROLLOUT_DOMAINS`. `HQ_SYNC_PRESIGN_TRANSPORT` overrides the email
241
+ * check in both directions (`1`/`true`/`yes`/`on` → force on,
242
+ * `0`/`false`/`no`/`off` → force off) so the transport can be exercised by
243
+ * unenrolled testers or rolled back for enrolled accounts without a redeploy.
244
+ * An unset/blank override falls through to the email check; an unrecognized
245
+ * override value is ignored (email check wins) rather than silently forcing
246
+ * a state.
223
247
  */
224
248
  export function resolvePresignTransport(
225
249
  email: string | undefined,
@@ -228,7 +252,10 @@ export function resolvePresignTransport(
228
252
  const o = (override ?? "").trim().toLowerCase();
229
253
  if (o === "1" || o === "true" || o === "yes" || o === "on") return true;
230
254
  if (o === "0" || o === "false" || o === "no" || o === "off") return false;
231
- return typeof email === "string" && email.toLowerCase().endsWith("@getindigo.ai");
255
+ if (typeof email !== "string") return false;
256
+ const at = email.lastIndexOf("@");
257
+ if (at < 0) return false;
258
+ return PRESIGN_ROLLOUT_DOMAINS.has(email.slice(at + 1).toLowerCase());
232
259
  }
233
260
 
234
261
  // Personal-vault scope (exclusion list + path computer) lives in
@@ -1898,7 +1925,23 @@ export async function runRunnerWithLoop(
1898
1925
  deps: RunnerLoopDeps = {},
1899
1926
  ): Promise<number> {
1900
1927
  if (!argv.includes("--watch")) {
1901
- return runRunner(argv);
1928
+ // One-shot cloud sync — take the per-root operation lock so it is mutually
1929
+ // exclusive with rescue/reindex. The `--watch` path below is the push
1930
+ // watcher and is intentionally EXEMPT (it neither takes nor is blocked by
1931
+ // the lock; its in-process targeted passes call `runRunner` directly, not
1932
+ // through here). If args don't parse, fall through to `runRunner` so it
1933
+ // surfaces the parse error rather than us masking it with a lock failure.
1934
+ const parsed = parseArgs(argv);
1935
+ if ("error" in parsed) return runRunner(argv);
1936
+ try {
1937
+ return await withOperationLock(parsed.hqRoot, "sync", () => runRunner(argv));
1938
+ } catch (err) {
1939
+ if (err instanceof OperationLockedError) {
1940
+ process.stderr.write(err.message + "\n");
1941
+ return OPERATION_LOCKED_EXIT;
1942
+ }
1943
+ throw err;
1944
+ }
1902
1945
  }
1903
1946
  const sleep =
1904
1947
  deps.sleep ??