@indigoai-us/hq-cloud 6.2.6 → 6.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +8 -7
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +51 -9
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +74 -2
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts +8 -0
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +222 -198
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +35 -0
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +39 -16
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.reindex.test.js +15 -2
- package/dist/cli/rescue.reindex.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +3 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/operation-lock.d.ts +100 -0
- package/dist/operation-lock.d.ts.map +1 -0
- package/dist/operation-lock.js +256 -0
- package/dist/operation-lock.js.map +1 -0
- package/dist/operation-lock.test.d.ts +5 -0
- package/dist/operation-lock.test.d.ts.map +1 -0
- package/dist/operation-lock.test.js +140 -0
- package/dist/operation-lock.test.js.map +1 -0
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +91 -2
- package/src/bin/sync-runner.ts +52 -9
- package/src/cli/reindex.test.ts +45 -0
- package/src/cli/reindex.ts +36 -0
- package/src/cli/rescue.reindex.test.ts +17 -2
- package/src/cli/rescue.ts +40 -15
- package/src/cli/sync.test.ts +2 -1
- package/src/cli/sync.ts +3 -1
- package/src/operation-lock.test.ts +162 -0
- package/src/operation-lock.ts +293 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-HQ-root mutual exclusion for the long-running operations
|
|
3
|
+
* (`sync`, `rescue`, `reindex`).
|
|
4
|
+
*
|
|
5
|
+
* Contract:
|
|
6
|
+
* - At most ONE of sync / rescue / reindex runs at a time **per HQ root**.
|
|
7
|
+
* The lock is shared across all three (keyed only by the root, not the
|
|
8
|
+
* command), so e.g. a rescue refuses while a sync holds it. Different HQ
|
|
9
|
+
* roots are fully independent — they hash to different lock files.
|
|
10
|
+
* - The push watcher / watch+event-push runner is EXEMPT: it never calls in
|
|
11
|
+
* here, so it neither takes the lock nor is blocked by it (its targeted
|
|
12
|
+
* in-process push passes are likewise lock-free).
|
|
13
|
+
*
|
|
14
|
+
* ## Where the lock lives — and why
|
|
15
|
+
*
|
|
16
|
+
* `<stateDir>/locks/operation-<hash(canonicalRoot)>.lock`, where
|
|
17
|
+
* `stateDir = $HQ_STATE_DIR || ~/.hq`. This is deliberately NOT inside the HQ
|
|
18
|
+
* root:
|
|
19
|
+
* - It must never round-trip to the cloud. A lock is machine-local, per-run
|
|
20
|
+
* state; syncing it to S3 (and thence to other machines/roots) would be a
|
|
21
|
+
* correctness bug. `~/.hq` is the established machine-local state dir
|
|
22
|
+
* (journals already live there) and is never synced.
|
|
23
|
+
* - `rescue` repairs a possibly-broken HQ root; a lock that depends on the
|
|
24
|
+
* root being healthy is exactly backwards. `~/.hq` is independent of the
|
|
25
|
+
* root's health.
|
|
26
|
+
* - Keying the filename by a hash of the *canonical* root path makes the
|
|
27
|
+
* lock per-root and prevents leakage across roots, while keeping the path
|
|
28
|
+
* short and filesystem-safe.
|
|
29
|
+
*
|
|
30
|
+
* ## Atomicity, liveness, takeover
|
|
31
|
+
*
|
|
32
|
+
* - Acquisition uses `open(…, "wx")` (O_CREAT | O_EXCL) — an atomic
|
|
33
|
+
* create-if-absent. Exactly one racer can create the file; the loser sees
|
|
34
|
+
* EEXIST and re-evaluates.
|
|
35
|
+
* - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
|
|
36
|
+
* EEXIST we test the recorded PID with `process.kill(pid, 0)`:
|
|
37
|
+
* * ESRCH → the holder is gone (crashed / killed -9 / stale file) →
|
|
38
|
+
* reclaim the lock.
|
|
39
|
+
* * EPERM → the PID exists but is owned by another user → treat as ALIVE
|
|
40
|
+
* (conservative: refuse rather than risk two concurrent ops).
|
|
41
|
+
* * success → alive → refuse fast with {@link OperationLockedError}
|
|
42
|
+
* naming the holding command + PID.
|
|
43
|
+
* - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
|
|
44
|
+
* the original holder crashed and the OS later handed its PID to an
|
|
45
|
+
* unrelated process, we conservatively read that as "still held" and
|
|
46
|
+
* refuse. We accept that false-busy over the far worse false-free, and
|
|
47
|
+
* record `startedAt`/`command` so an operator can diagnose a wedged lock.
|
|
48
|
+
*
|
|
49
|
+
* ## Release
|
|
50
|
+
*
|
|
51
|
+
* - Normal exit: the `with*` wrappers release in a `finally`.
|
|
52
|
+
* - Signals (SIGINT/SIGTERM): a one-time handler releases every held lock,
|
|
53
|
+
* then re-raises the default disposition so exit status is unchanged.
|
|
54
|
+
* - Hard crash (SIGKILL / power loss): nothing runs, but the stale-PID
|
|
55
|
+
* takeover above reclaims the lock on the next attempt.
|
|
56
|
+
* - `process.on("exit")`: a final best-effort synchronous unlink.
|
|
57
|
+
*
|
|
58
|
+
* ## Escape hatch
|
|
59
|
+
*
|
|
60
|
+
* `HQ_DISABLE_OP_LOCK=1` makes acquisition a no-op (returns a handle whose
|
|
61
|
+
* release does nothing). For emergencies and for callers that manage
|
|
62
|
+
* exclusion themselves; documented, off by default.
|
|
63
|
+
*/
|
|
64
|
+
import * as crypto from "crypto";
|
|
65
|
+
import * as fs from "fs";
|
|
66
|
+
import * as os from "os";
|
|
67
|
+
import * as path from "path";
|
|
68
|
+
/** Process exit code used when an operation is refused because the lock is held. */
|
|
69
|
+
export const OPERATION_LOCKED_EXIT = 17;
|
|
70
|
+
/** Thrown by `acquireOperationLock` when a LIVE holder owns the lock. */
|
|
71
|
+
export class OperationLockedError extends Error {
|
|
72
|
+
holder;
|
|
73
|
+
attempted;
|
|
74
|
+
constructor(holder, attempted) {
|
|
75
|
+
super(`Refusing to start "${attempted}": another HQ operation is already ` +
|
|
76
|
+
`running for this HQ root — "${holder.command}" (pid ${holder.pid}, ` +
|
|
77
|
+
`started ${holder.startedAt}). Wait for it to finish, or stop that ` +
|
|
78
|
+
`process, then retry.`);
|
|
79
|
+
this.holder = holder;
|
|
80
|
+
this.attempted = attempted;
|
|
81
|
+
this.name = "OperationLockedError";
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function stateDir() {
|
|
85
|
+
return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
|
|
86
|
+
}
|
|
87
|
+
/** Absolute lock path for a given HQ root. Exported for tests. */
|
|
88
|
+
export function lockPathFor(hqRoot) {
|
|
89
|
+
const canon = path.resolve(hqRoot);
|
|
90
|
+
const key = crypto.createHash("sha1").update(canon).digest("hex").slice(0, 16);
|
|
91
|
+
return path.join(stateDir(), "locks", `operation-${key}.lock`);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Is `pid` a live process? `kill(pid, 0)` sends no signal; it only probes.
|
|
95
|
+
* ESRCH → no such process (dead/stale). EPERM → exists but not ours → ALIVE
|
|
96
|
+
* (conservative). Anything else → assume alive rather than risk a double-run.
|
|
97
|
+
*/
|
|
98
|
+
function pidAlive(pid) {
|
|
99
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
100
|
+
return false;
|
|
101
|
+
try {
|
|
102
|
+
process.kill(pid, 0);
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
const code = err?.code;
|
|
107
|
+
if (code === "ESRCH")
|
|
108
|
+
return false;
|
|
109
|
+
return true; // EPERM (exists) or unknown → treat as alive
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function readLockInfo(p) {
|
|
113
|
+
try {
|
|
114
|
+
const parsed = JSON.parse(fs.readFileSync(p, "utf8"));
|
|
115
|
+
if (parsed && typeof parsed.pid === "number" && typeof parsed.command === "string") {
|
|
116
|
+
return parsed;
|
|
117
|
+
}
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
// ── Process-wide release plumbing ──────────────────────────────────────────
|
|
125
|
+
// Track every lock this process currently holds so the signal/exit hooks can
|
|
126
|
+
// release all of them. The hooks are installed exactly once.
|
|
127
|
+
const heldLocks = new Set();
|
|
128
|
+
let hooksInstalled = false;
|
|
129
|
+
function unlinkIfOwned(p) {
|
|
130
|
+
// Only remove a lock whose recorded pid is THIS process — never clobber a
|
|
131
|
+
// lock another process took over after a (hypothetical) reclaim race.
|
|
132
|
+
const info = readLockInfo(p);
|
|
133
|
+
if (info && info.pid === process.pid) {
|
|
134
|
+
try {
|
|
135
|
+
fs.unlinkSync(p);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
/* already gone — fine */
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function installHooksOnce() {
|
|
143
|
+
if (hooksInstalled)
|
|
144
|
+
return;
|
|
145
|
+
hooksInstalled = true;
|
|
146
|
+
process.on("exit", () => {
|
|
147
|
+
for (const h of heldLocks)
|
|
148
|
+
unlinkIfOwned(h.path);
|
|
149
|
+
});
|
|
150
|
+
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
|
151
|
+
process.on(sig, () => {
|
|
152
|
+
for (const h of heldLocks)
|
|
153
|
+
unlinkIfOwned(h.path);
|
|
154
|
+
// Re-raise with the default disposition so the exit status is the normal
|
|
155
|
+
// signal status (and a second Ctrl-C still works). Removing our listener
|
|
156
|
+
// first avoids recursing back into this handler.
|
|
157
|
+
process.removeAllListeners(sig);
|
|
158
|
+
process.kill(process.pid, sig);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
function makeHandle(p, info) {
|
|
163
|
+
const handle = {
|
|
164
|
+
path: p,
|
|
165
|
+
info,
|
|
166
|
+
release() {
|
|
167
|
+
heldLocks.delete(handle);
|
|
168
|
+
unlinkIfOwned(p);
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
heldLocks.add(handle);
|
|
172
|
+
installHooksOnce();
|
|
173
|
+
return handle;
|
|
174
|
+
}
|
|
175
|
+
const NOOP_HANDLE_BASE = { release() { } };
|
|
176
|
+
/**
|
|
177
|
+
* Acquire the per-root operation lock for `command`. Returns a {@link LockHandle}
|
|
178
|
+
* on success; throws {@link OperationLockedError} when a live holder owns it.
|
|
179
|
+
* Reclaims a stale lock (dead holder) transparently.
|
|
180
|
+
*/
|
|
181
|
+
export function acquireOperationLock(hqRoot, command) {
|
|
182
|
+
if (process.env.HQ_DISABLE_OP_LOCK === "1") {
|
|
183
|
+
const info = {
|
|
184
|
+
pid: process.pid,
|
|
185
|
+
command,
|
|
186
|
+
startedAt: new Date().toISOString(),
|
|
187
|
+
hqRoot: path.resolve(hqRoot),
|
|
188
|
+
};
|
|
189
|
+
return { ...NOOP_HANDLE_BASE, path: "", info };
|
|
190
|
+
}
|
|
191
|
+
const p = lockPathFor(hqRoot);
|
|
192
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
193
|
+
const info = {
|
|
194
|
+
pid: process.pid,
|
|
195
|
+
command,
|
|
196
|
+
startedAt: new Date().toISOString(),
|
|
197
|
+
hqRoot: path.resolve(hqRoot),
|
|
198
|
+
};
|
|
199
|
+
const payload = JSON.stringify(info, null, 2);
|
|
200
|
+
// Bounded retry: each iteration is one atomic create attempt. EEXIST against
|
|
201
|
+
// a stale holder reclaims and retries; EEXIST against a live holder refuses.
|
|
202
|
+
const MAX_ATTEMPTS = 5;
|
|
203
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
204
|
+
let fd;
|
|
205
|
+
try {
|
|
206
|
+
fd = fs.openSync(p, "wx"); // O_CREAT | O_EXCL — atomic
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
if (err?.code !== "EEXIST")
|
|
210
|
+
throw err;
|
|
211
|
+
const holder = readLockInfo(p);
|
|
212
|
+
if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
|
|
213
|
+
throw new OperationLockedError(holder, command);
|
|
214
|
+
}
|
|
215
|
+
// Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
|
|
216
|
+
try {
|
|
217
|
+
fs.unlinkSync(p);
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
/* someone else reclaimed it first; the next openSync re-evaluates */
|
|
221
|
+
}
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
try {
|
|
225
|
+
fs.writeSync(fd, payload);
|
|
226
|
+
}
|
|
227
|
+
finally {
|
|
228
|
+
fs.closeSync(fd);
|
|
229
|
+
}
|
|
230
|
+
return makeHandle(p, info);
|
|
231
|
+
}
|
|
232
|
+
// Pathological churn (another process reclaiming in lockstep). Surface it
|
|
233
|
+
// rather than spin forever.
|
|
234
|
+
throw new Error(`Could not acquire HQ operation lock at ${p} after ${MAX_ATTEMPTS} attempts`);
|
|
235
|
+
}
|
|
236
|
+
/** Run `fn` while holding the per-root lock for `command` (async). */
|
|
237
|
+
export async function withOperationLock(hqRoot, command, fn) {
|
|
238
|
+
const handle = acquireOperationLock(hqRoot, command);
|
|
239
|
+
try {
|
|
240
|
+
return await fn();
|
|
241
|
+
}
|
|
242
|
+
finally {
|
|
243
|
+
handle.release();
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/** Run `fn` while holding the per-root lock for `command` (synchronous). */
|
|
247
|
+
export function withOperationLockSync(hqRoot, command, fn) {
|
|
248
|
+
const handle = acquireOperationLock(hqRoot, command);
|
|
249
|
+
try {
|
|
250
|
+
return fn();
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
handle.release();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
//# sourceMappingURL=operation-lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operation-lock.js","sourceRoot":"","sources":["../src/operation-lock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,oFAAoF;AACpF,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAWxC,yEAAyE;AACzE,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAE3B;IACA;IAFlB,YACkB,MAAgB,EAChB,SAAiB;QAEjC,KAAK,CACH,sBAAsB,SAAS,qCAAqC;YAClE,+BAA+B,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,IAAI;YACrE,WAAW,MAAM,CAAC,SAAS,yCAAyC;YACpE,sBAAsB,CACzB,CAAC;QARc,WAAM,GAAN,MAAM,CAAU;QAChB,cAAS,GAAT,SAAS,CAAQ;QAQjC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAWD,SAAS,QAAQ;IACf,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;AACpE,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/E,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAI,GAA6B,EAAE,IAAI,CAAC;QAClD,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,IAAI,CAAC,CAAC,6CAA6C;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAa,CAAC;QAClE,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnF,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,6EAA6E;AAC7E,6DAA6D;AAE7D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc,CAAC;AACxC,IAAI,cAAc,GAAG,KAAK,CAAC;AAE3B,SAAS,aAAa,CAAC,CAAS;IAC9B,0EAA0E;IAC1E,sEAAsE;IACtE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,cAAc;QAAE,OAAO;IAC3B,cAAc,GAAG,IAAI,CAAC;IAEtB,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACtB,KAAK,MAAM,CAAC,IAAI,SAAS;YAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QAC3D,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE;YACnB,KAAK,MAAM,CAAC,IAAI,SAAS;gBAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjD,yEAAyE;YACzE,yEAAyE;YACzE,iDAAiD;YACjD,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAAS,EAAE,IAAc;IAC3C,MAAM,MAAM,GAAe;QACzB,IAAI,EAAE,CAAC;QACP,IAAI;QACJ,OAAO;YACL,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,aAAa,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;KACF,CAAC;IACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,gBAAgB,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,gBAAgB,GAAG,EAAE,OAAO,KAAI,CAAC,EAAE,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,OAAe;IAClE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAa;YACrB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO;YACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;SAC7B,CAAC;QACF,OAAO,EAAE,GAAG,gBAAgB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IACjD,CAAC;IAED,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC7B,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAE9C,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,YAAY,GAAG,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,EAAU,CAAC;QACf,IAAI,CAAC;YACH,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,4BAA4B;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,EAAE,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;YACD,uEAAuE;YACvE,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;YACvE,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,IAAI,KAAK,CACb,0CAA0C,CAAC,UAAU,YAAY,WAAW,CAC7E,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,OAAe,EACf,EAAoB;IAEpB,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,qBAAqB,CACnC,MAAc,EACd,OAAe,EACf,EAAW;IAEX,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operation-lock.test.d.ts","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the per-HQ-root operation mutex.
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
5
|
+
import { spawnSync } from "child_process";
|
|
6
|
+
import * as fs from "fs";
|
|
7
|
+
import * as os from "os";
|
|
8
|
+
import * as path from "path";
|
|
9
|
+
import { acquireOperationLock, withOperationLockSync, lockPathFor, OperationLockedError, OPERATION_LOCKED_EXIT, } from "./operation-lock.js";
|
|
10
|
+
/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
|
|
11
|
+
function deadPid() {
|
|
12
|
+
const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
|
|
13
|
+
if (!r.pid)
|
|
14
|
+
throw new Error("could not spawn to obtain a dead pid");
|
|
15
|
+
return r.pid;
|
|
16
|
+
}
|
|
17
|
+
function writeLock(p, info) {
|
|
18
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
19
|
+
const full = {
|
|
20
|
+
pid: 1,
|
|
21
|
+
command: "sync",
|
|
22
|
+
startedAt: new Date(0).toISOString(),
|
|
23
|
+
hqRoot: "/x",
|
|
24
|
+
...info,
|
|
25
|
+
};
|
|
26
|
+
fs.writeFileSync(p, JSON.stringify(full));
|
|
27
|
+
}
|
|
28
|
+
describe("operation-lock", () => {
|
|
29
|
+
let stateDir;
|
|
30
|
+
let rootA;
|
|
31
|
+
let rootB;
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
|
|
34
|
+
process.env.HQ_STATE_DIR = stateDir;
|
|
35
|
+
delete process.env.HQ_DISABLE_OP_LOCK;
|
|
36
|
+
rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
|
|
37
|
+
rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
|
|
38
|
+
});
|
|
39
|
+
afterEach(() => {
|
|
40
|
+
fs.rmSync(stateDir, { recursive: true, force: true });
|
|
41
|
+
fs.rmSync(rootA, { recursive: true, force: true });
|
|
42
|
+
fs.rmSync(rootB, { recursive: true, force: true });
|
|
43
|
+
delete process.env.HQ_STATE_DIR;
|
|
44
|
+
delete process.env.HQ_DISABLE_OP_LOCK;
|
|
45
|
+
});
|
|
46
|
+
it("the lock path is under the state dir, keyed per canonical root", () => {
|
|
47
|
+
const a = lockPathFor(rootA);
|
|
48
|
+
const b = lockPathFor(rootB);
|
|
49
|
+
expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
|
|
50
|
+
expect(a).not.toBe(b); // different roots → different lock files
|
|
51
|
+
// canonical: a trailing-slash variant maps to the same lock
|
|
52
|
+
expect(lockPathFor(rootA + path.sep)).toBe(a);
|
|
53
|
+
});
|
|
54
|
+
it("acquires, writes holder info, and releases (file gone after release)", () => {
|
|
55
|
+
const h = acquireOperationLock(rootA, "sync");
|
|
56
|
+
expect(fs.existsSync(h.path)).toBe(true);
|
|
57
|
+
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
58
|
+
expect(info.pid).toBe(process.pid);
|
|
59
|
+
expect(info.command).toBe("sync");
|
|
60
|
+
h.release();
|
|
61
|
+
expect(fs.existsSync(h.path)).toBe(false);
|
|
62
|
+
});
|
|
63
|
+
it("refuses fast with the holder's command + pid when a LIVE process holds it", () => {
|
|
64
|
+
// Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
|
|
65
|
+
// is always alive and is never our own pid, so kill(1,0) reports alive and
|
|
66
|
+
// the same-process reclaim path does not apply.
|
|
67
|
+
writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
|
|
68
|
+
expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
|
|
69
|
+
try {
|
|
70
|
+
acquireOperationLock(rootA, "sync");
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
const err = e;
|
|
74
|
+
expect(err.holder.command).toBe("rescue");
|
|
75
|
+
expect(err.holder.pid).toBe(1);
|
|
76
|
+
expect(err.message).toContain("rescue");
|
|
77
|
+
expect(err.message).toContain("pid 1");
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
it("reclaims a stale lock whose holder PID is dead (takeover)", () => {
|
|
81
|
+
const stale = deadPid();
|
|
82
|
+
writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
|
|
83
|
+
// The dead holder must not block us.
|
|
84
|
+
const h = acquireOperationLock(rootA, "rescue");
|
|
85
|
+
const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
|
|
86
|
+
expect(info.pid).toBe(process.pid); // we took it over
|
|
87
|
+
expect(info.command).toBe("rescue");
|
|
88
|
+
h.release();
|
|
89
|
+
});
|
|
90
|
+
it("reclaims a torn/unreadable lock file", () => {
|
|
91
|
+
const p = lockPathFor(rootA);
|
|
92
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
93
|
+
fs.writeFileSync(p, "{ this is not valid json");
|
|
94
|
+
const h = acquireOperationLock(rootA, "reindex");
|
|
95
|
+
expect(fs.existsSync(h.path)).toBe(true);
|
|
96
|
+
h.release();
|
|
97
|
+
});
|
|
98
|
+
it("different HQ roots are independent — both may hold concurrently", () => {
|
|
99
|
+
const a = acquireOperationLock(rootA, "sync");
|
|
100
|
+
const b = acquireOperationLock(rootB, "rescue"); // must NOT refuse
|
|
101
|
+
expect(fs.existsSync(a.path)).toBe(true);
|
|
102
|
+
expect(fs.existsSync(b.path)).toBe(true);
|
|
103
|
+
expect(a.path).not.toBe(b.path);
|
|
104
|
+
a.release();
|
|
105
|
+
b.release();
|
|
106
|
+
});
|
|
107
|
+
it("the same root is mutually exclusive across different commands", () => {
|
|
108
|
+
// A live sync in ANOTHER process holds the root (pid 1 stands in for it).
|
|
109
|
+
const p = lockPathFor(rootA);
|
|
110
|
+
writeLock(p, { pid: 1, command: "sync" });
|
|
111
|
+
// Neither rescue nor reindex may acquire while that sync holds it.
|
|
112
|
+
expect(() => acquireOperationLock(rootA, "rescue")).toThrowError(OperationLockedError);
|
|
113
|
+
expect(() => acquireOperationLock(rootA, "reindex")).toThrowError(OperationLockedError);
|
|
114
|
+
// Once that sync finishes (its lock is gone), the next command acquires.
|
|
115
|
+
fs.unlinkSync(p);
|
|
116
|
+
const h2 = acquireOperationLock(rootA, "reindex");
|
|
117
|
+
expect(fs.existsSync(h2.path)).toBe(true);
|
|
118
|
+
h2.release();
|
|
119
|
+
});
|
|
120
|
+
it("withOperationLockSync releases even when the body throws", () => {
|
|
121
|
+
const p = lockPathFor(rootA);
|
|
122
|
+
expect(() => withOperationLockSync(rootA, "reindex", () => {
|
|
123
|
+
expect(fs.existsSync(p)).toBe(true); // held during the body
|
|
124
|
+
throw new Error("boom");
|
|
125
|
+
})).toThrow("boom");
|
|
126
|
+
expect(fs.existsSync(p)).toBe(false); // released on the way out
|
|
127
|
+
});
|
|
128
|
+
it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
|
|
129
|
+
process.env.HQ_DISABLE_OP_LOCK = "1";
|
|
130
|
+
// Even with a live holder on record, the escape hatch acquires without error.
|
|
131
|
+
writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
|
|
132
|
+
const h = acquireOperationLock(rootA, "rescue");
|
|
133
|
+
expect(h.path).toBe(""); // no real lock file written
|
|
134
|
+
h.release(); // no-op, no throw
|
|
135
|
+
});
|
|
136
|
+
it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
|
|
137
|
+
expect(OPERATION_LOCKED_EXIT).toBe(17);
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
//# sourceMappingURL=operation-lock.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operation-lock.test.js","sourceRoot":"","sources":["../src/operation-lock.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,WAAW,EACX,oBAAoB,EACpB,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AAE7B,yFAAyF;AACzF,SAAS,OAAO;IACd,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,IAAuB;IACnD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa;QACrB,GAAG,EAAE,CAAC;QACN,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QACpC,MAAM,EAAE,IAAI;QACZ,GAAG,IAAI;KACR,CAAC;IACF,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,KAAa,CAAC;IAClB,IAAI,KAAa,CAAC;IAElB,UAAU,CAAC,GAAG,EAAE;QACd,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC;QACpC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACtC,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,yCAAyC;QAChE,4DAA4D;QAC5D,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC9E,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,2EAA2E;QAC3E,2EAA2E;QAC3E,gDAAgD;QAChD,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAyB,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC;QACxB,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/D,qCAAqC;QACrC,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAa,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,kBAAkB;QACnE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,0EAA0E;QAC1E,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,mEAAmE;QACnE,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACvF,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACxF,yEAAyE;QACzE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,EAAE,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,EAAE,CAAC,OAAO,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,EAAE,CACV,qBAAqB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YAC5D,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA0B;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,GAAG,CAAC;QACrC,8EAA8E;QAC9E,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACrD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,kBAAkB;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -30,6 +30,7 @@ import type {
|
|
|
30
30
|
} from "./sync-runner.js";
|
|
31
31
|
import { FakeClock } from "../watcher.js";
|
|
32
32
|
import { PERSONAL_VAULT_JOURNAL_SLUG } from "../journal.js";
|
|
33
|
+
import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
|
|
33
34
|
import type { SyncResult, SyncOptions } from "../cli/sync.js";
|
|
34
35
|
import type { ShareResult, ShareOptions } from "../cli/share.js";
|
|
35
36
|
import type {
|
|
@@ -3176,13 +3177,25 @@ describe("resolvePresignTransport", () => {
|
|
|
3176
3177
|
expect(resolvePresignTransport("ME@GetIndigo.AI", undefined)).toBe(true);
|
|
3177
3178
|
});
|
|
3178
3179
|
|
|
3179
|
-
it("
|
|
3180
|
+
it("ON for batch-2 rollout domains (gmail.com, vyg.ai, amass.com), no override", () => {
|
|
3181
|
+
expect(resolvePresignTransport("someone@gmail.com", undefined)).toBe(true);
|
|
3182
|
+
expect(resolvePresignTransport("Someone@GMAIL.com", undefined)).toBe(true);
|
|
3183
|
+
expect(resolvePresignTransport("shahzaib@vyg.ai", undefined)).toBe(true);
|
|
3184
|
+
expect(resolvePresignTransport("ops@amass.com", undefined)).toBe(true);
|
|
3185
|
+
});
|
|
3186
|
+
|
|
3187
|
+
it("OFF for unenrolled emails, no override", () => {
|
|
3180
3188
|
expect(resolvePresignTransport("me@example.com", undefined)).toBe(false);
|
|
3181
3189
|
expect(resolvePresignTransport(undefined, undefined)).toBe(false);
|
|
3182
|
-
|
|
3190
|
+
expect(resolvePresignTransport("no-at-sign", undefined)).toBe(false);
|
|
3191
|
+
// Exact-domain matching — lookalikes and suffix tricks must not match.
|
|
3183
3192
|
expect(resolvePresignTransport("me@notgetindigo.ai.evil.com", undefined)).toBe(
|
|
3184
3193
|
false,
|
|
3185
3194
|
);
|
|
3195
|
+
expect(resolvePresignTransport("me@gmail.com.evil.com", undefined)).toBe(false);
|
|
3196
|
+
expect(resolvePresignTransport("me@evil-gmail.com", undefined)).toBe(false);
|
|
3197
|
+
// ridge.com is deliberately NOT in batch 2.
|
|
3198
|
+
expect(resolvePresignTransport("juan@ridge.com", undefined)).toBe(false);
|
|
3186
3199
|
});
|
|
3187
3200
|
|
|
3188
3201
|
it.each(["1", "true", "yes", "on", "ON", " True "])(
|
|
@@ -3554,3 +3567,79 @@ describe("readPinnedPrefixes", () => {
|
|
|
3554
3567
|
expect(readPinnedPrefixes(root, "acme")).toEqual([]);
|
|
3555
3568
|
});
|
|
3556
3569
|
});
|
|
3570
|
+
|
|
3571
|
+
// ---------------------------------------------------------------------------
|
|
3572
|
+
// Operation lock — one-shot sync takes it; the watch runner is exempt.
|
|
3573
|
+
// ---------------------------------------------------------------------------
|
|
3574
|
+
describe("runRunnerWithLoop — operation lock", () => {
|
|
3575
|
+
const HQ = "/tmp/hq-oplock";
|
|
3576
|
+
|
|
3577
|
+
/** Write a live-holder lock (pid 1) for the HQ root into the test state dir. */
|
|
3578
|
+
function writeLiveHolder(command: string): string {
|
|
3579
|
+
const p = lockPathFor(HQ);
|
|
3580
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
3581
|
+
fs.writeFileSync(
|
|
3582
|
+
p,
|
|
3583
|
+
JSON.stringify({ pid: 1, command, startedAt: new Date(0).toISOString(), hqRoot: HQ }),
|
|
3584
|
+
);
|
|
3585
|
+
return p;
|
|
3586
|
+
}
|
|
3587
|
+
|
|
3588
|
+
it("one-shot sync refuses fast (exit 17) when another op holds the root", async () => {
|
|
3589
|
+
const lp = writeLiveHolder("rescue");
|
|
3590
|
+
const errs: string[] = [];
|
|
3591
|
+
const spy = vi
|
|
3592
|
+
.spyOn(process.stderr, "write")
|
|
3593
|
+
.mockImplementation((chunk: string | Uint8Array) => {
|
|
3594
|
+
errs.push(String(chunk));
|
|
3595
|
+
return true;
|
|
3596
|
+
});
|
|
3597
|
+
|
|
3598
|
+
// No --watch → one-shot. Refusal short-circuits BEFORE runRunner (so no
|
|
3599
|
+
// network / auth is touched).
|
|
3600
|
+
const code = await runRunnerWithLoop(["--companies", "--hq-root", HQ]);
|
|
3601
|
+
|
|
3602
|
+
spy.mockRestore();
|
|
3603
|
+
expect(code).toBe(OPERATION_LOCKED_EXIT);
|
|
3604
|
+
expect(errs.join("")).toContain("rescue"); // names the holder
|
|
3605
|
+
// The holder's lock is left intact — we refused, we didn't take it over.
|
|
3606
|
+
const held = JSON.parse(fs.readFileSync(lp, "utf8"));
|
|
3607
|
+
expect(held.pid).toBe(1);
|
|
3608
|
+
expect(held.command).toBe("rescue");
|
|
3609
|
+
});
|
|
3610
|
+
|
|
3611
|
+
it("the watch runner is EXEMPT — runs despite a held lock and never takes it", async () => {
|
|
3612
|
+
const lp = writeLiveHolder("sync");
|
|
3613
|
+
const watcher = makeWatcherStub();
|
|
3614
|
+
let triggerShutdown = () => {};
|
|
3615
|
+
const runPass = vi.fn().mockResolvedValue(0);
|
|
3616
|
+
|
|
3617
|
+
const loop = runRunnerWithLoop(
|
|
3618
|
+
["--companies", "--watch", "--event-push", "--hq-root", HQ],
|
|
3619
|
+
{
|
|
3620
|
+
runPass,
|
|
3621
|
+
clock: new FakeClock(),
|
|
3622
|
+
createWatcher: () => watcher,
|
|
3623
|
+
sleep: () => new Promise<void>(() => {}),
|
|
3624
|
+
onShutdownSignal: (handler) => {
|
|
3625
|
+
triggerShutdown = handler;
|
|
3626
|
+
return () => {};
|
|
3627
|
+
},
|
|
3628
|
+
},
|
|
3629
|
+
);
|
|
3630
|
+
|
|
3631
|
+
await Promise.resolve();
|
|
3632
|
+
await Promise.resolve();
|
|
3633
|
+
// It started and ran a pass even though the lock is held → not blocked.
|
|
3634
|
+
expect(watcher.started).toBe(true);
|
|
3635
|
+
expect(runPass).toHaveBeenCalled();
|
|
3636
|
+
|
|
3637
|
+
triggerShutdown();
|
|
3638
|
+
await loop;
|
|
3639
|
+
|
|
3640
|
+
// The pre-existing holder lock is untouched → the watcher never took it.
|
|
3641
|
+
const held = JSON.parse(fs.readFileSync(lp, "utf8"));
|
|
3642
|
+
expect(held.pid).toBe(1);
|
|
3643
|
+
expect(held.command).toBe("sync");
|
|
3644
|
+
});
|
|
3645
|
+
});
|
package/src/bin/sync-runner.ts
CHANGED
|
@@ -100,6 +100,11 @@ import { collectAndSendTelemetry } from "../telemetry.js";
|
|
|
100
100
|
import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
|
|
101
101
|
import { reindexAfterSync } from "../qmd-reindex.js";
|
|
102
102
|
import { pruneConflictIndex } from "../lib/conflict-index.js";
|
|
103
|
+
import {
|
|
104
|
+
withOperationLock,
|
|
105
|
+
OperationLockedError,
|
|
106
|
+
OPERATION_LOCKED_EXIT,
|
|
107
|
+
} from "../operation-lock.js";
|
|
103
108
|
import { describeError } from "../lib/describe-error.js";
|
|
104
109
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
105
110
|
import {
|
|
@@ -210,16 +215,35 @@ export function resolveSkipPersonal(flag: boolean): boolean {
|
|
|
210
215
|
return env === "1" || env === "true" || env === "yes";
|
|
211
216
|
}
|
|
212
217
|
|
|
218
|
+
/**
|
|
219
|
+
* Email domains enrolled in the presigned-URL transport rollout.
|
|
220
|
+
*
|
|
221
|
+
* Staged by the operator (2026-06-10) in descending user count: getindigo.ai
|
|
222
|
+
* was the pilot; gmail.com / vyg.ai / amass.com are batch 2. The presign
|
|
223
|
+
* transport also routes every upload through the vault API's
|
|
224
|
+
* `validateObjectKey` (INVALID_KEY_BACKSLASH et al.), so enrolling a domain
|
|
225
|
+
* upgrades its server-side input validation versus the STS-direct path.
|
|
226
|
+
* Matching is on the EXACT domain (the part after the last `@`), not a
|
|
227
|
+
* suffix — `evil-gmail.com` and `gmail.com.evil.com` never match.
|
|
228
|
+
*/
|
|
229
|
+
const PRESIGN_ROLLOUT_DOMAINS: ReadonlySet<string> = new Set([
|
|
230
|
+
"getindigo.ai",
|
|
231
|
+
"gmail.com",
|
|
232
|
+
"vyg.ai",
|
|
233
|
+
"amass.com",
|
|
234
|
+
]);
|
|
235
|
+
|
|
213
236
|
/**
|
|
214
237
|
* Decide whether this session uses the presigned-URL transport.
|
|
215
238
|
*
|
|
216
|
-
* Rollout gate: ON for accounts whose verified email is
|
|
217
|
-
* `HQ_SYNC_PRESIGN_TRANSPORT` overrides the email
|
|
218
|
-
* (`1`/`true`/`yes`/`on` → force on,
|
|
219
|
-
* the transport can be exercised by
|
|
220
|
-
*
|
|
221
|
-
* to the email check; an unrecognized
|
|
222
|
-
* wins) rather than silently forcing
|
|
239
|
+
* Rollout gate: ON for accounts whose verified email domain is enrolled in
|
|
240
|
+
* `PRESIGN_ROLLOUT_DOMAINS`. `HQ_SYNC_PRESIGN_TRANSPORT` overrides the email
|
|
241
|
+
* check in both directions (`1`/`true`/`yes`/`on` → force on,
|
|
242
|
+
* `0`/`false`/`no`/`off` → force off) so the transport can be exercised by
|
|
243
|
+
* unenrolled testers or rolled back for enrolled accounts without a redeploy.
|
|
244
|
+
* An unset/blank override falls through to the email check; an unrecognized
|
|
245
|
+
* override value is ignored (email check wins) rather than silently forcing
|
|
246
|
+
* a state.
|
|
223
247
|
*/
|
|
224
248
|
export function resolvePresignTransport(
|
|
225
249
|
email: string | undefined,
|
|
@@ -228,7 +252,10 @@ export function resolvePresignTransport(
|
|
|
228
252
|
const o = (override ?? "").trim().toLowerCase();
|
|
229
253
|
if (o === "1" || o === "true" || o === "yes" || o === "on") return true;
|
|
230
254
|
if (o === "0" || o === "false" || o === "no" || o === "off") return false;
|
|
231
|
-
|
|
255
|
+
if (typeof email !== "string") return false;
|
|
256
|
+
const at = email.lastIndexOf("@");
|
|
257
|
+
if (at < 0) return false;
|
|
258
|
+
return PRESIGN_ROLLOUT_DOMAINS.has(email.slice(at + 1).toLowerCase());
|
|
232
259
|
}
|
|
233
260
|
|
|
234
261
|
// Personal-vault scope (exclusion list + path computer) lives in
|
|
@@ -1898,7 +1925,23 @@ export async function runRunnerWithLoop(
|
|
|
1898
1925
|
deps: RunnerLoopDeps = {},
|
|
1899
1926
|
): Promise<number> {
|
|
1900
1927
|
if (!argv.includes("--watch")) {
|
|
1901
|
-
|
|
1928
|
+
// One-shot cloud sync — take the per-root operation lock so it is mutually
|
|
1929
|
+
// exclusive with rescue/reindex. The `--watch` path below is the push
|
|
1930
|
+
// watcher and is intentionally EXEMPT (it neither takes nor is blocked by
|
|
1931
|
+
// the lock; its in-process targeted passes call `runRunner` directly, not
|
|
1932
|
+
// through here). If args don't parse, fall through to `runRunner` so it
|
|
1933
|
+
// surfaces the parse error rather than us masking it with a lock failure.
|
|
1934
|
+
const parsed = parseArgs(argv);
|
|
1935
|
+
if ("error" in parsed) return runRunner(argv);
|
|
1936
|
+
try {
|
|
1937
|
+
return await withOperationLock(parsed.hqRoot, "sync", () => runRunner(argv));
|
|
1938
|
+
} catch (err) {
|
|
1939
|
+
if (err instanceof OperationLockedError) {
|
|
1940
|
+
process.stderr.write(err.message + "\n");
|
|
1941
|
+
return OPERATION_LOCKED_EXIT;
|
|
1942
|
+
}
|
|
1943
|
+
throw err;
|
|
1944
|
+
}
|
|
1902
1945
|
}
|
|
1903
1946
|
const sleep =
|
|
1904
1947
|
deps.sleep ??
|