@indigoai-us/hq-cloud 6.14.18 → 6.14.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +35 -0
- package/dist/bin/sync-runner-company.d.ts +7 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +21 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +10 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +201 -25
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +14 -2
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +17 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +22 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +495 -3
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-core.js +27 -1
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-drop-dir-symlink.test.d.ts +2 -0
- package/dist/cli/rescue-drop-dir-symlink.test.d.ts.map +1 -0
- package/dist/cli/rescue-drop-dir-symlink.test.js +206 -0
- package/dist/cli/rescue-drop-dir-symlink.test.js.map +1 -0
- package/dist/cli/share.d.ts +15 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -55
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +119 -6
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +19 -25
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +121 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +3 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +13 -2
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +26 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/personal-vault.d.ts +12 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +20 -0
- package/dist/personal-vault.js.map +1 -1
- package/dist/qmd-reindex.d.ts +88 -11
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +354 -65
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/qmd-reindex.test.d.ts +4 -0
- package/dist/qmd-reindex.test.d.ts.map +1 -1
- package/dist/qmd-reindex.test.js +285 -3
- package/dist/qmd-reindex.test.js.map +1 -1
- package/dist/s3.d.ts +13 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +34 -1
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +60 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/telemetry.d.ts +22 -0
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +79 -0
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +117 -1
- package/dist/telemetry.test.js.map +1 -1
- package/dist/watcher.d.ts +26 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +75 -12
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +29 -1
- package/src/bin/sync-runner-watch-loop.ts +322 -32
- package/src/bin/sync-runner-watch-routes.ts +14 -1
- package/src/bin/sync-runner.test.ts +591 -6
- package/src/bin/sync-runner.ts +50 -5
- package/src/cli/rescue-core.ts +26 -1
- package/src/cli/rescue-drop-dir-symlink.test.ts +224 -0
- package/src/cli/share.test.ts +150 -6
- package/src/cli/share.ts +143 -48
- package/src/cli/sync.test.ts +150 -1
- package/src/cli/sync.ts +25 -29
- package/src/journal.test.ts +45 -0
- package/src/journal.ts +18 -1
- package/src/personal-vault.ts +23 -0
- package/src/qmd-reindex.test.ts +324 -4
- package/src/qmd-reindex.ts +414 -76
- package/src/s3.test.ts +79 -0
- package/src/s3.ts +48 -1
- package/src/telemetry.test.ts +128 -0
- package/src/telemetry.ts +80 -0
- package/src/watcher.ts +111 -14
- package/test/e2e/watcher-real-chokidar.test.ts +62 -2
- package/test/e2e/watcher-recursive-backend.test.ts +67 -1
package/src/qmd-reindex.ts
CHANGED
|
@@ -24,27 +24,293 @@
|
|
|
24
24
|
*
|
|
25
25
|
* The index itself is never synced — it is large, binary, and embeds absolute
|
|
26
26
|
* local paths. Only its *freshness* is automated here.
|
|
27
|
+
*
|
|
28
|
+
* ## Corruption safety (feedback_b9a369ff)
|
|
29
|
+
*
|
|
30
|
+
* The qmd store (`.qmd/index.sqlite`, backed by a sqlite-vec virtual table) was
|
|
31
|
+
* being corrupted on the vector side (`content_vectors` / `vectors_vec_rowids`)
|
|
32
|
+
* because TWO uncoordinated writers raced on it: this post-sync reindex and an
|
|
33
|
+
* interactive session's own qmd maintenance (e.g. a skill's `qmd embed`). Two
|
|
34
|
+
* defects compounded:
|
|
35
|
+
* 1. A hard 120s `spawnSync` timeout that KILLED a long `qmd update`/`embed`
|
|
36
|
+
* mid-write, tearing down the process between vector-table writes.
|
|
37
|
+
* 2. No cross-process serialization, so an interactive `qmd embed` and a
|
|
38
|
+
* runner-spawned `qmd update` overlapped with near certainty.
|
|
39
|
+
*
|
|
40
|
+
* The mitigations here, all inside the runner so every teammate gets them on
|
|
41
|
+
* their next sync:
|
|
42
|
+
* - The exec timeout is raised substantially so a legitimate long pass is no
|
|
43
|
+
* longer killed mid-write in the first place. A timed-out pass is treated as
|
|
44
|
+
* "not done" (left dirty, retried next cycle) and NEVER advances to embed.
|
|
45
|
+
* That not-done handling — together with the writer serialization below — is
|
|
46
|
+
* the real protection; the signal used for the (now rare) kill is secondary.
|
|
47
|
+
* When the bound does fire the child is sent SIGINT rather than the default
|
|
48
|
+
* SIGTERM, but qmd 2.5.3 traps both identically (each just restores the
|
|
49
|
+
* cursor and exits), so this is only the same clean exit path qmd takes on
|
|
50
|
+
* Ctrl-C — not a guarantee the in-flight SQLite write unwinds.
|
|
51
|
+
* - A shared advisory lock at `<hqRoot>/.qmd/.reindex.lock` serializes qmd
|
|
52
|
+
* writers. The runner takes it before touching qmd and SKIPS the cycle (and
|
|
53
|
+
* marks the tree dirty so the next sync retries) when another writer holds
|
|
54
|
+
* it. The lock path sits next to the DB so interactive HQ qmd maintenance
|
|
55
|
+
* can honor the same rendezvous.
|
|
56
|
+
* - Corruption is detected cheaply from qmd's own output (the SQLITE_CORRUPT
|
|
57
|
+
* signature). On detection the corrupt DB files are quarantined aside (moved
|
|
58
|
+
* to `index.sqlite.corrupt-<ts>`, never deleted) and the cycle stops writing
|
|
59
|
+
* so a clean rebuild happens on the next pass — instead of writing further
|
|
60
|
+
* onto an already-damaged store.
|
|
27
61
|
*/
|
|
28
62
|
|
|
29
63
|
import * as fs from "fs";
|
|
30
64
|
import * as path from "path";
|
|
31
65
|
import { spawnSync } from "child_process";
|
|
32
66
|
|
|
67
|
+
/** Result of a single `qmd` invocation. */
|
|
68
|
+
export interface QmdExecResult {
|
|
69
|
+
status: number | null;
|
|
70
|
+
stdout: string;
|
|
71
|
+
/** Captured stderr (qmd prints SQLITE_CORRUPT errors here). Optional for fakes. */
|
|
72
|
+
stderr?: string;
|
|
73
|
+
/** True when the process was killed because it exceeded the exec timeout. */
|
|
74
|
+
timedOut?: boolean;
|
|
75
|
+
}
|
|
76
|
+
|
|
33
77
|
/** Injectable command runner — real `spawnSync` in prod, a fake in tests. */
|
|
34
78
|
export interface QmdExec {
|
|
35
|
-
(args: string[]):
|
|
79
|
+
(args: string[]): QmdExecResult;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Default exec timeout for a single `qmd` invocation. The historical 120s bound
|
|
84
|
+
* routinely killed `qmd update`/`qmd embed` mid-write on a large index (the HQ
|
|
85
|
+
* root's `hq` collection alone spans the whole tree), which corrupted the vector
|
|
86
|
+
* store. 15 minutes comfortably covers a cold or embed-heavy pass, so the bound
|
|
87
|
+
* effectively stops firing on legitimate work; a genuinely wedged process is
|
|
88
|
+
* still bounded, and a timed-out pass is treated as not-done (never written
|
|
89
|
+
* further onto) — see {@link defaultExec} and {@link reindexAfterSync}.
|
|
90
|
+
*/
|
|
91
|
+
export const DEFAULT_QMD_EXEC_TIMEOUT_MS = 900_000;
|
|
92
|
+
|
|
93
|
+
/** Resolve the exec timeout, honoring `HQ_QMD_EXEC_TIMEOUT_MS` (ms) if valid. */
|
|
94
|
+
export function resolveExecTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
95
|
+
const raw = env.HQ_QMD_EXEC_TIMEOUT_MS;
|
|
96
|
+
if (raw !== undefined && raw !== "") {
|
|
97
|
+
const n = Number(raw);
|
|
98
|
+
if (Number.isFinite(n) && n > 0) return Math.round(n);
|
|
99
|
+
}
|
|
100
|
+
return DEFAULT_QMD_EXEC_TIMEOUT_MS;
|
|
36
101
|
}
|
|
37
102
|
|
|
38
103
|
const defaultExec: QmdExec = (args) => {
|
|
39
104
|
const res = spawnSync("qmd", args, {
|
|
40
105
|
encoding: "utf8",
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
timeout:
|
|
106
|
+
// Bound so a wedged index never hangs the runner forever, but generously —
|
|
107
|
+
// see DEFAULT_QMD_EXEC_TIMEOUT_MS.
|
|
108
|
+
timeout: resolveExecTimeoutMs(),
|
|
109
|
+
// On timeout send SIGINT instead of the default SIGTERM. NOTE: qmd 2.5.3
|
|
110
|
+
// traps BOTH identically — each handler only restores the cursor and calls
|
|
111
|
+
// process.exit() — so SIGINT is NOT inherently a graceful transaction
|
|
112
|
+
// unwind; it is simply the same exit path qmd takes on Ctrl-C. The real
|
|
113
|
+
// write-safety guarantee is the raised bound above (a kill is now rare) plus
|
|
114
|
+
// treating a timed-out pass as not-done and never embedding after it (see
|
|
115
|
+
// reindexAfterSync). SIGINT is kept only so a killed pass exits qmd's normal
|
|
116
|
+
// way rather than via an unhandled default signal.
|
|
117
|
+
killSignal: "SIGINT",
|
|
118
|
+
});
|
|
119
|
+
const timedOut =
|
|
120
|
+
res.error != null &&
|
|
121
|
+
(res.error as NodeJS.ErrnoException).code === "ETIMEDOUT";
|
|
122
|
+
return {
|
|
123
|
+
status: res.status,
|
|
124
|
+
stdout: res.stdout ?? "",
|
|
125
|
+
stderr: res.stderr ?? "",
|
|
126
|
+
timedOut,
|
|
127
|
+
};
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/** SQLite corruption signature qmd surfaces when the vector store is damaged. */
|
|
131
|
+
const CORRUPTION_SIGNATURE = /SQLITE_CORRUPT|database disk image is malformed/i;
|
|
132
|
+
|
|
133
|
+
/** True if a qmd result reports SQLite corruption on stdout or stderr. */
|
|
134
|
+
export function looksCorrupt(r: Partial<Pick<QmdExecResult, "stdout" | "stderr">>): boolean {
|
|
135
|
+
return CORRUPTION_SIGNATURE.test(`${r.stdout ?? ""}\n${r.stderr ?? ""}`);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Handle returned by an acquired reindex lock. `release()` is idempotent. */
|
|
139
|
+
export interface ReindexLockHandle {
|
|
140
|
+
release(): void;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Acquire the shared reindex lock at `lockPath`. Returns a handle when acquired,
|
|
145
|
+
* or `null` when another live writer already holds it (caller should skip the
|
|
146
|
+
* cycle). Must never throw — coordination is advisory and must not fail a sync.
|
|
147
|
+
*/
|
|
148
|
+
export type AcquireReindexLock = (lockPath: string) => ReindexLockHandle | null;
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* How long a lock file may sit before a new acquirer treats it as abandoned and
|
|
152
|
+
* reclaims it, even if its recorded PID can't be probed (e.g. a different user,
|
|
153
|
+
* or a torn/empty file). Backstops the PID-liveness check for the crash case.
|
|
154
|
+
*/
|
|
155
|
+
const DEFAULT_LOCK_STALE_MS = 30 * 60_000;
|
|
156
|
+
|
|
157
|
+
function reindexLockStaleMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
158
|
+
const raw = env.HQ_QMD_LOCK_STALE_MS;
|
|
159
|
+
if (raw !== undefined && raw !== "") {
|
|
160
|
+
const n = Number(raw);
|
|
161
|
+
if (Number.isFinite(n) && n >= 0) return Math.round(n);
|
|
162
|
+
}
|
|
163
|
+
return DEFAULT_LOCK_STALE_MS;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const NOOP_LOCK: ReindexLockHandle = { release() {} };
|
|
167
|
+
|
|
168
|
+
// Track locks this process holds so a clean exit/signal removes them; a crash is
|
|
169
|
+
// covered by the stale/PID reclaim path in the acquirer.
|
|
170
|
+
const heldReindexLocks = new Set<string>();
|
|
171
|
+
let reindexExitHookInstalled = false;
|
|
172
|
+
|
|
173
|
+
function installReindexExitHookOnce(): void {
|
|
174
|
+
if (reindexExitHookInstalled) return;
|
|
175
|
+
reindexExitHookInstalled = true;
|
|
176
|
+
process.on("exit", () => {
|
|
177
|
+
for (const p of heldReindexLocks) unlinkIfOwned(p);
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Is `pid` a live process? ESRCH → dead; EPERM/other → conservatively alive. */
|
|
182
|
+
function pidAlive(pid: number): boolean {
|
|
183
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
184
|
+
try {
|
|
185
|
+
process.kill(pid, 0);
|
|
186
|
+
return true;
|
|
187
|
+
} catch (err) {
|
|
188
|
+
return (err as NodeJS.ErrnoException)?.code !== "ESRCH";
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Unlink `lockPath` only if it still records THIS process as the holder. */
|
|
193
|
+
function unlinkIfOwned(lockPath: string): void {
|
|
194
|
+
try {
|
|
195
|
+
const info = JSON.parse(fs.readFileSync(lockPath, "utf8")) as { pid?: number };
|
|
196
|
+
if (info?.pid !== process.pid) return;
|
|
197
|
+
} catch {
|
|
198
|
+
// Unreadable/torn/already-gone — don't risk clobbering another holder.
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
try {
|
|
202
|
+
fs.unlinkSync(lockPath);
|
|
203
|
+
} catch {
|
|
204
|
+
/* already gone — fine */
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* If the lock at `lockPath` is held by a dead PID or is older than the stale
|
|
210
|
+
* bound, unlink it so the caller can retry. Returns true when it reclaimed (or
|
|
211
|
+
* the lock vanished mid-check), false when a live, fresh holder still owns it.
|
|
212
|
+
*/
|
|
213
|
+
function reclaimReindexLockIfStale(lockPath: string): boolean {
|
|
214
|
+
let st: fs.Stats;
|
|
215
|
+
try {
|
|
216
|
+
st = fs.statSync(lockPath);
|
|
217
|
+
} catch {
|
|
218
|
+
// Vanished between EEXIST and stat → the holder released; let caller retry.
|
|
219
|
+
return true;
|
|
220
|
+
}
|
|
221
|
+
let holderPid = 0;
|
|
222
|
+
try {
|
|
223
|
+
const info = JSON.parse(fs.readFileSync(lockPath, "utf8")) as { pid?: number };
|
|
224
|
+
if (typeof info?.pid === "number") holderPid = info.pid;
|
|
225
|
+
} catch {
|
|
226
|
+
/* torn/empty lock — fall through to the staleness check */
|
|
227
|
+
}
|
|
228
|
+
const dead = holderPid > 0 && !pidAlive(holderPid);
|
|
229
|
+
const stale = Date.now() - st.mtimeMs > reindexLockStaleMs();
|
|
230
|
+
if (dead || stale) {
|
|
231
|
+
try {
|
|
232
|
+
fs.unlinkSync(lockPath);
|
|
233
|
+
} catch {
|
|
234
|
+
// Someone else reclaimed it first; the next create attempt re-evaluates.
|
|
235
|
+
}
|
|
236
|
+
return true;
|
|
237
|
+
}
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const defaultAcquireReindexLock: AcquireReindexLock = (lockPath) => {
|
|
242
|
+
// Escape hatch: a caller that manages exclusion itself can disable the lock.
|
|
243
|
+
if (process.env.HQ_QMD_REINDEX_LOCK === "0") return NOOP_LOCK;
|
|
244
|
+
|
|
245
|
+
const payload = JSON.stringify({
|
|
246
|
+
pid: process.pid,
|
|
247
|
+
startedAt: new Date().toISOString(),
|
|
248
|
+
command: "hq-cloud qmd-reindex",
|
|
44
249
|
});
|
|
45
|
-
|
|
250
|
+
|
|
251
|
+
// At most one reclaim + one retry: acquire, or (if stale) reclaim then retry.
|
|
252
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
253
|
+
try {
|
|
254
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
255
|
+
const fd = fs.openSync(lockPath, "wx", 0o600);
|
|
256
|
+
try {
|
|
257
|
+
fs.writeSync(fd, payload);
|
|
258
|
+
} finally {
|
|
259
|
+
fs.closeSync(fd);
|
|
260
|
+
}
|
|
261
|
+
heldReindexLocks.add(lockPath);
|
|
262
|
+
installReindexExitHookOnce();
|
|
263
|
+
return {
|
|
264
|
+
release() {
|
|
265
|
+
heldReindexLocks.delete(lockPath);
|
|
266
|
+
unlinkIfOwned(lockPath);
|
|
267
|
+
},
|
|
268
|
+
};
|
|
269
|
+
} catch (err) {
|
|
270
|
+
if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") {
|
|
271
|
+
// The `.qmd` dir isn't writable (unusual — it holds the DB). Fail OPEN
|
|
272
|
+
// rather than block the reindex on an infra problem: proceed without the
|
|
273
|
+
// advisory lock. The corruption guard below is the remaining backstop.
|
|
274
|
+
return NOOP_LOCK;
|
|
275
|
+
}
|
|
276
|
+
// Held. Reclaim iff the holder is dead/stale, then retry once.
|
|
277
|
+
if (reclaimReindexLockIfStale(lockPath)) continue;
|
|
278
|
+
return null; // live, fresh holder → skip this cycle
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
return null;
|
|
46
282
|
};
|
|
47
283
|
|
|
284
|
+
/** Move corrupt DB files aside (never delete) so a clean rebuild can follow. */
|
|
285
|
+
export type QuarantineCorruptIndex = (
|
|
286
|
+
qmdDir: string,
|
|
287
|
+
timestampSuffix: string,
|
|
288
|
+
) => string | null;
|
|
289
|
+
|
|
290
|
+
const CORRUPTIBLE_DB_FILES = ["index.sqlite", "index.sqlite-wal", "index.sqlite-shm"];
|
|
291
|
+
|
|
292
|
+
const defaultQuarantineCorruptIndex: QuarantineCorruptIndex = (qmdDir, tsSuffix) => {
|
|
293
|
+
let movedBase: string | null = null;
|
|
294
|
+
for (const name of CORRUPTIBLE_DB_FILES) {
|
|
295
|
+
const src = path.join(qmdDir, name);
|
|
296
|
+
try {
|
|
297
|
+
if (!fs.existsSync(src)) continue;
|
|
298
|
+
const dest = `${src}.corrupt-${tsSuffix}`;
|
|
299
|
+
fs.renameSync(src, dest);
|
|
300
|
+
if (name === "index.sqlite") movedBase = dest;
|
|
301
|
+
} catch {
|
|
302
|
+
// Best-effort: move what we can. A file we can't move is left in place;
|
|
303
|
+
// the corruption guard still prevents further writes this cycle.
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return movedBase;
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
/** Filesystem-safe timestamp suffix for a quarantined DB file. */
|
|
310
|
+
function quarantineSuffix(nowMs: number): string {
|
|
311
|
+
return new Date(nowMs).toISOString().replace(/[:.]/g, "-");
|
|
312
|
+
}
|
|
313
|
+
|
|
48
314
|
const DEFAULT_CHANGED_PATH_DEBOUNCE_MS = 60_000;
|
|
49
315
|
|
|
50
316
|
export interface ReindexOptions {
|
|
@@ -71,6 +337,10 @@ export interface ReindexOptions {
|
|
|
71
337
|
readCompanies?: (companiesDir: string) => string[];
|
|
72
338
|
/** Returns true if the knowledge dir has at least one indexable .md file. */
|
|
73
339
|
hasIndexableMarkdown?: (knowledgeDir: string) => boolean;
|
|
340
|
+
/** Reindex-lock acquirer override for tests. */
|
|
341
|
+
acquireReindexLock?: AcquireReindexLock;
|
|
342
|
+
/** Corrupt-index quarantine override for tests. */
|
|
343
|
+
quarantineCorruptIndex?: QuarantineCorruptIndex;
|
|
74
344
|
/** Optional diagnostic sink for unexpected swallowed failures. */
|
|
75
345
|
log?: (diagnostic: {
|
|
76
346
|
event: string;
|
|
@@ -80,6 +350,20 @@ export interface ReindexOptions {
|
|
|
80
350
|
}) => void;
|
|
81
351
|
}
|
|
82
352
|
|
|
353
|
+
export interface ReindexResult {
|
|
354
|
+
qmdAvailable: boolean;
|
|
355
|
+
collectionsAdded: string[];
|
|
356
|
+
updated: boolean;
|
|
357
|
+
embedded: boolean;
|
|
358
|
+
pendingDirty: boolean;
|
|
359
|
+
/** True when the cycle was skipped because another writer held the lock. */
|
|
360
|
+
lockBusy: boolean;
|
|
361
|
+
/** True when a qmd command exceeded the exec timeout and was aborted. */
|
|
362
|
+
timedOut: boolean;
|
|
363
|
+
/** True when a corrupt index was detected and quarantined this cycle. */
|
|
364
|
+
corruptionQuarantined: boolean;
|
|
365
|
+
}
|
|
366
|
+
|
|
83
367
|
/**
|
|
84
368
|
* Reindex qmd for an HQ tree after a sync. Never throws — all failures are
|
|
85
369
|
* swallowed so a reindex problem can never mask or fail the sync result.
|
|
@@ -89,21 +373,20 @@ export interface ReindexOptions {
|
|
|
89
373
|
export function reindexAfterSync(
|
|
90
374
|
hqRoot: string,
|
|
91
375
|
opts: ReindexOptions = {},
|
|
92
|
-
): {
|
|
93
|
-
qmdAvailable: boolean;
|
|
94
|
-
collectionsAdded: string[];
|
|
95
|
-
updated: boolean;
|
|
96
|
-
embedded: boolean;
|
|
97
|
-
pendingDirty: boolean;
|
|
98
|
-
} {
|
|
376
|
+
): ReindexResult {
|
|
99
377
|
const exec = opts.exec ?? defaultExec;
|
|
100
378
|
const existsSync = opts.existsSync ?? fs.existsSync;
|
|
101
|
-
const
|
|
379
|
+
const acquireLock = opts.acquireReindexLock ?? defaultAcquireReindexLock;
|
|
380
|
+
const quarantine = opts.quarantineCorruptIndex ?? defaultQuarantineCorruptIndex;
|
|
381
|
+
const result: ReindexResult = {
|
|
102
382
|
qmdAvailable: false,
|
|
103
|
-
collectionsAdded: []
|
|
383
|
+
collectionsAdded: [],
|
|
104
384
|
updated: false,
|
|
105
385
|
embedded: false,
|
|
106
386
|
pendingDirty: false,
|
|
387
|
+
lockBusy: false,
|
|
388
|
+
timedOut: false,
|
|
389
|
+
corruptionQuarantined: false,
|
|
107
390
|
};
|
|
108
391
|
|
|
109
392
|
try {
|
|
@@ -125,75 +408,109 @@ export function reindexAfterSync(
|
|
|
125
408
|
return result;
|
|
126
409
|
}
|
|
127
410
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
//
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
if (existingCollections.includes(`qmd://${slug}/`)) continue;
|
|
146
|
-
|
|
147
|
-
const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
|
|
148
|
-
if (add.status === 0) {
|
|
149
|
-
exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
|
|
150
|
-
result.collectionsAdded.push(slug);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
411
|
+
const nowMs = opts.nowMs ?? Date.now();
|
|
412
|
+
const pendingSinceMs =
|
|
413
|
+
pendingDirty && typeof state.pendingSinceMs === "number"
|
|
414
|
+
? state.pendingSinceMs
|
|
415
|
+
: nowMs;
|
|
416
|
+
|
|
417
|
+
// Serialize qmd writers. If another writer (this runner from a prior cycle,
|
|
418
|
+
// or an interactive session's qmd maintenance) holds the lock, SKIP this
|
|
419
|
+
// cycle and mark the tree dirty so the next sync retries — never write onto
|
|
420
|
+
// the vector store concurrently (feedback_b9a369ff).
|
|
421
|
+
const lockPath = path.join(hqRoot, ".qmd", ".reindex.lock");
|
|
422
|
+
const lock = acquireLock(lockPath);
|
|
423
|
+
if (!lock) {
|
|
424
|
+
writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
|
|
425
|
+
result.lockBusy = true;
|
|
426
|
+
result.pendingDirty = true;
|
|
427
|
+
return result;
|
|
153
428
|
}
|
|
154
429
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
pendingDirty && typeof state.pendingSinceMs === "number"
|
|
164
|
-
? state.pendingSinceMs
|
|
165
|
-
: nowMs;
|
|
166
|
-
if (debounceMs > 0 && nowMs - pendingSinceMs < debounceMs) {
|
|
167
|
-
writeState(statePath, {
|
|
168
|
-
...state,
|
|
169
|
-
pendingDirty: true,
|
|
170
|
-
pendingSinceMs,
|
|
171
|
-
});
|
|
172
|
-
result.pendingDirty = true;
|
|
430
|
+
try {
|
|
431
|
+
// Guard: qmd must be installed. `qmd collection list` doubles as the
|
|
432
|
+
// availability probe AND the source for which collections already exist.
|
|
433
|
+
const list = exec(["collection", "list"]);
|
|
434
|
+
if (list.status !== 0) return result; // qmd absent or errored — no-op
|
|
435
|
+
// An already-corrupt store can surface here even on the lexical side.
|
|
436
|
+
if (looksCorrupt(list)) {
|
|
437
|
+
handleCorruption(hqRoot, statePath, state, pendingSinceMs, nowMs, quarantine, result);
|
|
173
438
|
return result;
|
|
174
439
|
}
|
|
440
|
+
result.qmdAvailable = true;
|
|
441
|
+
const existingCollections = list.stdout;
|
|
442
|
+
|
|
443
|
+
// 1. Auto-register missing company knowledge collections.
|
|
444
|
+
if (registrationMayBeStale) {
|
|
445
|
+
const companiesDir = path.join(hqRoot, "companies");
|
|
446
|
+
const slugs = (opts.readCompanies ?? defaultReadCompanies)(companiesDir);
|
|
447
|
+
for (const slug of slugs) {
|
|
448
|
+
const knowledgeDir = path.join(companiesDir, slug, "knowledge");
|
|
449
|
+
if (!existsSync(knowledgeDir)) continue;
|
|
450
|
+
const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
|
|
451
|
+
if (!hasMd) continue;
|
|
452
|
+
// Already registered? qmd collection URIs look like `qmd://<slug>/`.
|
|
453
|
+
if (existingCollections.includes(`qmd://${slug}/`)) continue;
|
|
175
454
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
});
|
|
183
|
-
} else {
|
|
184
|
-
writeState(statePath, {
|
|
185
|
-
...state,
|
|
186
|
-
pendingDirty: true,
|
|
187
|
-
pendingSinceMs,
|
|
188
|
-
});
|
|
189
|
-
result.pendingDirty = true;
|
|
455
|
+
const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
|
|
456
|
+
if (add.status === 0) {
|
|
457
|
+
exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
|
|
458
|
+
result.collectionsAdded.push(slug);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
190
461
|
}
|
|
191
|
-
}
|
|
192
462
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
463
|
+
// 2. Incremental lexical reindex.
|
|
464
|
+
const shouldUpdate = dirtyFromChanges || pendingDirty;
|
|
465
|
+
if (shouldUpdate) {
|
|
466
|
+
const debounceMs =
|
|
467
|
+
opts.debounceMs ??
|
|
468
|
+
(changedPaths === undefined ? 0 : DEFAULT_CHANGED_PATH_DEBOUNCE_MS);
|
|
469
|
+
if (debounceMs > 0 && nowMs - pendingSinceMs < debounceMs) {
|
|
470
|
+
writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
|
|
471
|
+
result.pendingDirty = true;
|
|
472
|
+
return result;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
const update = exec(["update"]);
|
|
476
|
+
if (looksCorrupt(update)) {
|
|
477
|
+
handleCorruption(hqRoot, statePath, state, pendingSinceMs, nowMs, quarantine, result);
|
|
478
|
+
return result;
|
|
479
|
+
}
|
|
480
|
+
if (update.timedOut) {
|
|
481
|
+
// Aborted mid-pass by the timeout. Do NOT treat as done and do NOT
|
|
482
|
+
// proceed to embed — mark dirty and retry cleanly next cycle.
|
|
483
|
+
writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
|
|
484
|
+
result.timedOut = true;
|
|
485
|
+
result.pendingDirty = true;
|
|
486
|
+
return result;
|
|
487
|
+
}
|
|
488
|
+
result.updated = update.status === 0;
|
|
489
|
+
if (result.updated) {
|
|
490
|
+
writeState(statePath, { pendingDirty: false, lastSuccessMs: nowMs });
|
|
491
|
+
} else {
|
|
492
|
+
writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
|
|
493
|
+
result.pendingDirty = true;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// 3. Embeddings only on explicit request.
|
|
498
|
+
if (opts.embed) {
|
|
499
|
+
const embed = exec(["embed"]);
|
|
500
|
+
if (looksCorrupt(embed)) {
|
|
501
|
+
handleCorruption(hqRoot, statePath, state, pendingSinceMs, nowMs, quarantine, result);
|
|
502
|
+
return result;
|
|
503
|
+
}
|
|
504
|
+
if (embed.timedOut) {
|
|
505
|
+
writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
|
|
506
|
+
result.timedOut = true;
|
|
507
|
+
result.pendingDirty = true;
|
|
508
|
+
return result;
|
|
509
|
+
}
|
|
510
|
+
result.embedded = embed.status === 0;
|
|
511
|
+
}
|
|
512
|
+
} finally {
|
|
513
|
+
lock.release();
|
|
197
514
|
}
|
|
198
515
|
} catch (err) {
|
|
199
516
|
try {
|
|
@@ -212,6 +529,27 @@ export function reindexAfterSync(
|
|
|
212
529
|
return result;
|
|
213
530
|
}
|
|
214
531
|
|
|
532
|
+
/**
|
|
533
|
+
* Quarantine the corrupt DB aside and mark the tree dirty so the NEXT cycle
|
|
534
|
+
* rebuilds from scratch under the lock, instead of writing further onto an
|
|
535
|
+
* already-damaged vector store. Mutates `result` in place.
|
|
536
|
+
*/
|
|
537
|
+
function handleCorruption(
|
|
538
|
+
hqRoot: string,
|
|
539
|
+
statePath: string,
|
|
540
|
+
state: QmdReindexState,
|
|
541
|
+
pendingSinceMs: number,
|
|
542
|
+
nowMs: number,
|
|
543
|
+
quarantine: QuarantineCorruptIndex,
|
|
544
|
+
result: ReindexResult,
|
|
545
|
+
): void {
|
|
546
|
+
const qmdDir = path.join(hqRoot, ".qmd");
|
|
547
|
+
quarantine(qmdDir, quarantineSuffix(nowMs));
|
|
548
|
+
writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
|
|
549
|
+
result.corruptionQuarantined = true;
|
|
550
|
+
result.pendingDirty = true;
|
|
551
|
+
}
|
|
552
|
+
|
|
215
553
|
interface QmdReindexState {
|
|
216
554
|
pendingDirty?: boolean;
|
|
217
555
|
pendingSinceMs?: number;
|
package/src/s3.test.ts
CHANGED
|
@@ -95,6 +95,7 @@ import {
|
|
|
95
95
|
FILE_BTIME_META_KEY,
|
|
96
96
|
classifyVaultKey,
|
|
97
97
|
validateVaultUploadKey,
|
|
98
|
+
createStagedSymlink,
|
|
98
99
|
replaceStagedPath,
|
|
99
100
|
sweepStaleStagedFiles,
|
|
100
101
|
} from "./s3.js";
|
|
@@ -733,6 +734,84 @@ describe("uploadSymlink", () => {
|
|
|
733
734
|
});
|
|
734
735
|
});
|
|
735
736
|
|
|
737
|
+
describe("createStagedSymlink", () => {
|
|
738
|
+
it("uses an absolute junction for a Windows directory without requiring symlink privilege", () => {
|
|
739
|
+
const linkPath = path.join(path.sep, "vault", ".agents", "skills");
|
|
740
|
+
const target = "../.claude/skills";
|
|
741
|
+
const absTarget = path.resolve(path.dirname(linkPath), target);
|
|
742
|
+
const symlink = vi.fn(
|
|
743
|
+
(_target: string, _linkPath: string, type?: fs.symlink.Type) => {
|
|
744
|
+
if (type !== "junction") {
|
|
745
|
+
throw Object.assign(new Error("operation not permitted, symlink"), {
|
|
746
|
+
code: "EPERM",
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
},
|
|
750
|
+
);
|
|
751
|
+
|
|
752
|
+
expect(() =>
|
|
753
|
+
createStagedSymlink(target, linkPath, {
|
|
754
|
+
platform: "win32",
|
|
755
|
+
symlink,
|
|
756
|
+
statIsDirectory: () => true,
|
|
757
|
+
}),
|
|
758
|
+
).not.toThrow();
|
|
759
|
+
expect(symlink).toHaveBeenCalledWith(absTarget, linkPath, "junction");
|
|
760
|
+
expect(path.isAbsolute(symlink.mock.calls[0]![0])).toBe(true);
|
|
761
|
+
});
|
|
762
|
+
|
|
763
|
+
it("resolves a relative Windows overlay target before creating a junction", () => {
|
|
764
|
+
const linkPath = path.join(path.sep, "vault", ".codex", "claude");
|
|
765
|
+
const target = "../.claude/skills";
|
|
766
|
+
const symlink = vi.fn();
|
|
767
|
+
|
|
768
|
+
createStagedSymlink(target, linkPath, {
|
|
769
|
+
platform: "win32",
|
|
770
|
+
symlink,
|
|
771
|
+
statIsDirectory: () => undefined,
|
|
772
|
+
});
|
|
773
|
+
|
|
774
|
+
expect(symlink).toHaveBeenCalledWith(
|
|
775
|
+
path.resolve(path.dirname(linkPath), target),
|
|
776
|
+
linkPath,
|
|
777
|
+
"junction",
|
|
778
|
+
);
|
|
779
|
+
});
|
|
780
|
+
|
|
781
|
+
it("preserves a relative target and omits the type outside Windows", () => {
|
|
782
|
+
const linkPath = path.join(path.sep, "vault", ".agents", "skills");
|
|
783
|
+
const target = "../.claude/skills";
|
|
784
|
+
const symlink = vi.fn();
|
|
785
|
+
const statIsDirectory = vi.fn(() => true);
|
|
786
|
+
|
|
787
|
+
createStagedSymlink(target, linkPath, {
|
|
788
|
+
platform: "linux",
|
|
789
|
+
symlink,
|
|
790
|
+
statIsDirectory,
|
|
791
|
+
});
|
|
792
|
+
|
|
793
|
+
expect(symlink).toHaveBeenCalledWith(target, linkPath);
|
|
794
|
+
expect(statIsDirectory).not.toHaveBeenCalled();
|
|
795
|
+
});
|
|
796
|
+
|
|
797
|
+
it("uses a genuine Windows file symlink for a regular-file target", () => {
|
|
798
|
+
const linkPath = path.join(path.sep, "vault", "config-link.json");
|
|
799
|
+
const target = "../shared/config.json";
|
|
800
|
+
const absTarget = path.resolve(path.dirname(linkPath), target);
|
|
801
|
+
const symlink = vi.fn();
|
|
802
|
+
const statIsDirectory = vi.fn(() => false);
|
|
803
|
+
|
|
804
|
+
createStagedSymlink(target, linkPath, {
|
|
805
|
+
platform: "win32",
|
|
806
|
+
symlink,
|
|
807
|
+
statIsDirectory,
|
|
808
|
+
});
|
|
809
|
+
|
|
810
|
+
expect(statIsDirectory).toHaveBeenCalledWith(absTarget);
|
|
811
|
+
expect(symlink).toHaveBeenCalledWith(target, linkPath, "file");
|
|
812
|
+
});
|
|
813
|
+
});
|
|
814
|
+
|
|
736
815
|
describe("downloadFile", () => {
|
|
737
816
|
let tmpRoot: string;
|
|
738
817
|
|