@indigoai-us/hq-cloud 6.14.18 → 6.14.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/.github/workflows/ci.yml +35 -0
  2. package/dist/bin/sync-runner-company.d.ts +7 -0
  3. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  4. package/dist/bin/sync-runner-company.js +21 -1
  5. package/dist/bin/sync-runner-company.js.map +1 -1
  6. package/dist/bin/sync-runner-watch-loop.d.ts +10 -2
  7. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  8. package/dist/bin/sync-runner-watch-loop.js +201 -25
  9. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  10. package/dist/bin/sync-runner-watch-routes.d.ts +1 -1
  11. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  12. package/dist/bin/sync-runner-watch-routes.js +14 -2
  13. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  14. package/dist/bin/sync-runner.d.ts +17 -2
  15. package/dist/bin/sync-runner.d.ts.map +1 -1
  16. package/dist/bin/sync-runner.js +22 -4
  17. package/dist/bin/sync-runner.js.map +1 -1
  18. package/dist/bin/sync-runner.test.js +495 -3
  19. package/dist/bin/sync-runner.test.js.map +1 -1
  20. package/dist/cli/rescue-core.js +27 -1
  21. package/dist/cli/rescue-core.js.map +1 -1
  22. package/dist/cli/rescue-drop-dir-symlink.test.d.ts +2 -0
  23. package/dist/cli/rescue-drop-dir-symlink.test.d.ts.map +1 -0
  24. package/dist/cli/rescue-drop-dir-symlink.test.js +206 -0
  25. package/dist/cli/rescue-drop-dir-symlink.test.js.map +1 -0
  26. package/dist/cli/share.d.ts +15 -0
  27. package/dist/cli/share.d.ts.map +1 -1
  28. package/dist/cli/share.js +100 -55
  29. package/dist/cli/share.js.map +1 -1
  30. package/dist/cli/share.test.js +119 -6
  31. package/dist/cli/share.test.js.map +1 -1
  32. package/dist/cli/sync.d.ts.map +1 -1
  33. package/dist/cli/sync.js +19 -25
  34. package/dist/cli/sync.js.map +1 -1
  35. package/dist/cli/sync.test.js +121 -0
  36. package/dist/cli/sync.test.js.map +1 -1
  37. package/dist/journal.d.ts +3 -1
  38. package/dist/journal.d.ts.map +1 -1
  39. package/dist/journal.js +13 -2
  40. package/dist/journal.js.map +1 -1
  41. package/dist/journal.test.js +26 -1
  42. package/dist/journal.test.js.map +1 -1
  43. package/dist/personal-vault.d.ts +12 -0
  44. package/dist/personal-vault.d.ts.map +1 -1
  45. package/dist/personal-vault.js +20 -0
  46. package/dist/personal-vault.js.map +1 -1
  47. package/dist/qmd-reindex.d.ts +88 -11
  48. package/dist/qmd-reindex.d.ts.map +1 -1
  49. package/dist/qmd-reindex.js +354 -65
  50. package/dist/qmd-reindex.js.map +1 -1
  51. package/dist/qmd-reindex.test.d.ts +4 -0
  52. package/dist/qmd-reindex.test.d.ts.map +1 -1
  53. package/dist/qmd-reindex.test.js +285 -3
  54. package/dist/qmd-reindex.test.js.map +1 -1
  55. package/dist/s3.d.ts +13 -0
  56. package/dist/s3.d.ts.map +1 -1
  57. package/dist/s3.js +34 -1
  58. package/dist/s3.js.map +1 -1
  59. package/dist/s3.test.js +60 -1
  60. package/dist/s3.test.js.map +1 -1
  61. package/dist/telemetry.d.ts +22 -0
  62. package/dist/telemetry.d.ts.map +1 -1
  63. package/dist/telemetry.js +79 -0
  64. package/dist/telemetry.js.map +1 -1
  65. package/dist/telemetry.test.js +117 -1
  66. package/dist/telemetry.test.js.map +1 -1
  67. package/dist/watcher.d.ts +26 -1
  68. package/dist/watcher.d.ts.map +1 -1
  69. package/dist/watcher.js +75 -12
  70. package/dist/watcher.js.map +1 -1
  71. package/package.json +1 -1
  72. package/src/bin/sync-runner-company.ts +29 -1
  73. package/src/bin/sync-runner-watch-loop.ts +322 -32
  74. package/src/bin/sync-runner-watch-routes.ts +14 -1
  75. package/src/bin/sync-runner.test.ts +591 -6
  76. package/src/bin/sync-runner.ts +50 -5
  77. package/src/cli/rescue-core.ts +26 -1
  78. package/src/cli/rescue-drop-dir-symlink.test.ts +224 -0
  79. package/src/cli/share.test.ts +150 -6
  80. package/src/cli/share.ts +143 -48
  81. package/src/cli/sync.test.ts +150 -1
  82. package/src/cli/sync.ts +25 -29
  83. package/src/journal.test.ts +45 -0
  84. package/src/journal.ts +18 -1
  85. package/src/personal-vault.ts +23 -0
  86. package/src/qmd-reindex.test.ts +324 -4
  87. package/src/qmd-reindex.ts +414 -76
  88. package/src/s3.test.ts +79 -0
  89. package/src/s3.ts +48 -1
  90. package/src/telemetry.test.ts +128 -0
  91. package/src/telemetry.ts +80 -0
  92. package/src/watcher.ts +111 -14
  93. package/test/e2e/watcher-real-chokidar.test.ts +62 -2
  94. package/test/e2e/watcher-recursive-backend.test.ts +67 -1
@@ -24,27 +24,293 @@
24
24
  *
25
25
  * The index itself is never synced — it is large, binary, and embeds absolute
26
26
  * local paths. Only its *freshness* is automated here.
27
+ *
28
+ * ## Corruption safety (feedback_b9a369ff)
29
+ *
30
+ * The qmd store (`.qmd/index.sqlite`, backed by a sqlite-vec virtual table) was
31
+ * being corrupted on the vector side (`content_vectors` / `vectors_vec_rowids`)
32
+ * because TWO uncoordinated writers raced on it: this post-sync reindex and an
33
+ * interactive session's own qmd maintenance (e.g. a skill's `qmd embed`). Two
34
+ * defects compounded:
35
+ * 1. A hard 120s `spawnSync` timeout that KILLED a long `qmd update`/`embed`
36
+ * mid-write, tearing down the process between vector-table writes.
37
+ * 2. No cross-process serialization, so an interactive `qmd embed` and a
38
+ * runner-spawned `qmd update` overlapped with near certainty.
39
+ *
40
+ * The mitigations here, all inside the runner so every teammate gets them on
41
+ * their next sync:
42
+ * - The exec timeout is raised substantially so a legitimate long pass is no
43
+ * longer killed mid-write in the first place. A timed-out pass is treated as
44
+ * "not done" (left dirty, retried next cycle) and NEVER advances to embed.
45
+ * That not-done handling — together with the writer serialization below — is
46
+ * the real protection; the signal used for the (now rare) kill is secondary.
47
+ * When the bound does fire the child is sent SIGINT rather than the default
48
+ * SIGTERM, but qmd 2.5.3 traps both identically (each just restores the
49
+ * cursor and exits), so this is only the same clean exit path qmd takes on
50
+ * Ctrl-C — not a guarantee the in-flight SQLite write unwinds.
51
+ * - A shared advisory lock at `<hqRoot>/.qmd/.reindex.lock` serializes qmd
52
+ * writers. The runner takes it before touching qmd and SKIPS the cycle (and
53
+ * marks the tree dirty so the next sync retries) when another writer holds
54
+ * it. The lock path sits next to the DB so interactive HQ qmd maintenance
55
+ * can honor the same rendezvous.
56
+ * - Corruption is detected cheaply from qmd's own output (the SQLITE_CORRUPT
57
+ * signature). On detection the corrupt DB files are quarantined aside (moved
58
+ * to `index.sqlite.corrupt-<ts>`, never deleted) and the cycle stops writing
59
+ * so a clean rebuild happens on the next pass — instead of writing further
60
+ * onto an already-damaged store.
27
61
  */
28
62
 
29
63
  import * as fs from "fs";
30
64
  import * as path from "path";
31
65
  import { spawnSync } from "child_process";
32
66
 
67
+ /** Result of a single `qmd` invocation. */
68
+ export interface QmdExecResult {
69
+ status: number | null;
70
+ stdout: string;
71
+ /** Captured stderr (qmd prints SQLITE_CORRUPT errors here). Optional for fakes. */
72
+ stderr?: string;
73
+ /** True when the process was killed because it exceeded the exec timeout. */
74
+ timedOut?: boolean;
75
+ }
76
+
33
77
  /** Injectable command runner — real `spawnSync` in prod, a fake in tests. */
34
78
  export interface QmdExec {
35
- (args: string[]): { status: number | null; stdout: string };
79
+ (args: string[]): QmdExecResult;
80
+ }
81
+
82
+ /**
83
+ * Default exec timeout for a single `qmd` invocation. The historical 120s bound
84
+ * routinely killed `qmd update`/`qmd embed` mid-write on a large index (the HQ
85
+ * root's `hq` collection alone spans the whole tree), which corrupted the vector
86
+ * store. 15 minutes comfortably covers a cold or embed-heavy pass, so the bound
87
+ * effectively stops firing on legitimate work; a genuinely wedged process is
88
+ * still bounded, and a timed-out pass is treated as not-done (never written
89
+ * further onto) — see {@link defaultExec} and {@link reindexAfterSync}.
90
+ */
91
+ export const DEFAULT_QMD_EXEC_TIMEOUT_MS = 900_000;
92
+
93
+ /** Resolve the exec timeout, honoring `HQ_QMD_EXEC_TIMEOUT_MS` (ms) if valid. */
94
+ export function resolveExecTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
95
+ const raw = env.HQ_QMD_EXEC_TIMEOUT_MS;
96
+ if (raw !== undefined && raw !== "") {
97
+ const n = Number(raw);
98
+ if (Number.isFinite(n) && n > 0) return Math.round(n);
99
+ }
100
+ return DEFAULT_QMD_EXEC_TIMEOUT_MS;
36
101
  }
37
102
 
38
103
  const defaultExec: QmdExec = (args) => {
39
104
  const res = spawnSync("qmd", args, {
40
105
  encoding: "utf8",
41
- // qmd update on a large index can take a while; bound it so a wedged
42
- // index never hangs the runner forever.
43
- timeout: 120_000,
106
+ // Bound so a wedged index never hangs the runner forever, but generously —
107
+ // see DEFAULT_QMD_EXEC_TIMEOUT_MS.
108
+ timeout: resolveExecTimeoutMs(),
109
+ // On timeout send SIGINT instead of the default SIGTERM. NOTE: qmd 2.5.3
110
+ // traps BOTH identically — each handler only restores the cursor and calls
111
+ // process.exit() — so SIGINT is NOT inherently a graceful transaction
112
+ // unwind; it is simply the same exit path qmd takes on Ctrl-C. The real
113
+ // write-safety guarantee is the raised bound above (a kill is now rare) plus
114
+ // treating a timed-out pass as not-done and never embedding after it (see
115
+ // reindexAfterSync). SIGINT is kept only so a killed pass exits qmd's normal
116
+ // way rather than via an unhandled default signal.
117
+ killSignal: "SIGINT",
118
+ });
119
+ const timedOut =
120
+ res.error != null &&
121
+ (res.error as NodeJS.ErrnoException).code === "ETIMEDOUT";
122
+ return {
123
+ status: res.status,
124
+ stdout: res.stdout ?? "",
125
+ stderr: res.stderr ?? "",
126
+ timedOut,
127
+ };
128
+ };
129
+
130
+ /** SQLite corruption signature qmd surfaces when the vector store is damaged. */
131
+ const CORRUPTION_SIGNATURE = /SQLITE_CORRUPT|database disk image is malformed/i;
132
+
133
+ /** True if a qmd result reports SQLite corruption on stdout or stderr. */
134
+ export function looksCorrupt(r: Partial<Pick<QmdExecResult, "stdout" | "stderr">>): boolean {
135
+ return CORRUPTION_SIGNATURE.test(`${r.stdout ?? ""}\n${r.stderr ?? ""}`);
136
+ }
137
+
138
+ /** Handle returned by an acquired reindex lock. `release()` is idempotent. */
139
+ export interface ReindexLockHandle {
140
+ release(): void;
141
+ }
142
+
143
+ /**
144
+ * Acquire the shared reindex lock at `lockPath`. Returns a handle when acquired,
145
+ * or `null` when another live writer already holds it (caller should skip the
146
+ * cycle). Must never throw — coordination is advisory and must not fail a sync.
147
+ */
148
+ export type AcquireReindexLock = (lockPath: string) => ReindexLockHandle | null;
149
+
150
+ /**
151
+ * How long a lock file may sit before a new acquirer treats it as abandoned and
152
+ * reclaims it, even if its recorded PID can't be probed (e.g. a different user,
153
+ * or a torn/empty file). Backstops the PID-liveness check for the crash case.
154
+ */
155
+ const DEFAULT_LOCK_STALE_MS = 30 * 60_000;
156
+
157
+ function reindexLockStaleMs(env: NodeJS.ProcessEnv = process.env): number {
158
+ const raw = env.HQ_QMD_LOCK_STALE_MS;
159
+ if (raw !== undefined && raw !== "") {
160
+ const n = Number(raw);
161
+ if (Number.isFinite(n) && n >= 0) return Math.round(n);
162
+ }
163
+ return DEFAULT_LOCK_STALE_MS;
164
+ }
165
+
166
+ const NOOP_LOCK: ReindexLockHandle = { release() {} };
167
+
168
+ // Track locks this process holds so a clean exit/signal removes them; a crash is
169
+ // covered by the stale/PID reclaim path in the acquirer.
170
+ const heldReindexLocks = new Set<string>();
171
+ let reindexExitHookInstalled = false;
172
+
173
+ function installReindexExitHookOnce(): void {
174
+ if (reindexExitHookInstalled) return;
175
+ reindexExitHookInstalled = true;
176
+ process.on("exit", () => {
177
+ for (const p of heldReindexLocks) unlinkIfOwned(p);
178
+ });
179
+ }
180
+
181
+ /** Is `pid` a live process? ESRCH → dead; EPERM/other → conservatively alive. */
182
+ function pidAlive(pid: number): boolean {
183
+ if (!Number.isInteger(pid) || pid <= 0) return false;
184
+ try {
185
+ process.kill(pid, 0);
186
+ return true;
187
+ } catch (err) {
188
+ return (err as NodeJS.ErrnoException)?.code !== "ESRCH";
189
+ }
190
+ }
191
+
192
+ /** Unlink `lockPath` only if it still records THIS process as the holder. */
193
+ function unlinkIfOwned(lockPath: string): void {
194
+ try {
195
+ const info = JSON.parse(fs.readFileSync(lockPath, "utf8")) as { pid?: number };
196
+ if (info?.pid !== process.pid) return;
197
+ } catch {
198
+ // Unreadable/torn/already-gone — don't risk clobbering another holder.
199
+ return;
200
+ }
201
+ try {
202
+ fs.unlinkSync(lockPath);
203
+ } catch {
204
+ /* already gone — fine */
205
+ }
206
+ }
207
+
208
+ /**
209
+ * If the lock at `lockPath` is held by a dead PID or is older than the stale
210
+ * bound, unlink it so the caller can retry. Returns true when it reclaimed (or
211
+ * the lock vanished mid-check), false when a live, fresh holder still owns it.
212
+ */
213
+ function reclaimReindexLockIfStale(lockPath: string): boolean {
214
+ let st: fs.Stats;
215
+ try {
216
+ st = fs.statSync(lockPath);
217
+ } catch {
218
+ // Vanished between EEXIST and stat → the holder released; let caller retry.
219
+ return true;
220
+ }
221
+ let holderPid = 0;
222
+ try {
223
+ const info = JSON.parse(fs.readFileSync(lockPath, "utf8")) as { pid?: number };
224
+ if (typeof info?.pid === "number") holderPid = info.pid;
225
+ } catch {
226
+ /* torn/empty lock — fall through to the staleness check */
227
+ }
228
+ const dead = holderPid > 0 && !pidAlive(holderPid);
229
+ const stale = Date.now() - st.mtimeMs > reindexLockStaleMs();
230
+ if (dead || stale) {
231
+ try {
232
+ fs.unlinkSync(lockPath);
233
+ } catch {
234
+ // Someone else reclaimed it first; the next create attempt re-evaluates.
235
+ }
236
+ return true;
237
+ }
238
+ return false;
239
+ }
240
+
241
+ const defaultAcquireReindexLock: AcquireReindexLock = (lockPath) => {
242
+ // Escape hatch: a caller that manages exclusion itself can disable the lock.
243
+ if (process.env.HQ_QMD_REINDEX_LOCK === "0") return NOOP_LOCK;
244
+
245
+ const payload = JSON.stringify({
246
+ pid: process.pid,
247
+ startedAt: new Date().toISOString(),
248
+ command: "hq-cloud qmd-reindex",
44
249
  });
45
- return { status: res.status, stdout: res.stdout ?? "" };
250
+
251
+ // At most one reclaim + one retry: acquire, or (if stale) reclaim then retry.
252
+ for (let attempt = 0; attempt < 2; attempt++) {
253
+ try {
254
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
255
+ const fd = fs.openSync(lockPath, "wx", 0o600);
256
+ try {
257
+ fs.writeSync(fd, payload);
258
+ } finally {
259
+ fs.closeSync(fd);
260
+ }
261
+ heldReindexLocks.add(lockPath);
262
+ installReindexExitHookOnce();
263
+ return {
264
+ release() {
265
+ heldReindexLocks.delete(lockPath);
266
+ unlinkIfOwned(lockPath);
267
+ },
268
+ };
269
+ } catch (err) {
270
+ if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") {
271
+ // The `.qmd` dir isn't writable (unusual — it holds the DB). Fail OPEN
272
+ // rather than block the reindex on an infra problem: proceed without the
273
+ // advisory lock. The corruption guard below is the remaining backstop.
274
+ return NOOP_LOCK;
275
+ }
276
+ // Held. Reclaim iff the holder is dead/stale, then retry once.
277
+ if (reclaimReindexLockIfStale(lockPath)) continue;
278
+ return null; // live, fresh holder → skip this cycle
279
+ }
280
+ }
281
+ return null;
46
282
  };
47
283
 
284
+ /** Move corrupt DB files aside (never delete) so a clean rebuild can follow. */
285
+ export type QuarantineCorruptIndex = (
286
+ qmdDir: string,
287
+ timestampSuffix: string,
288
+ ) => string | null;
289
+
290
+ const CORRUPTIBLE_DB_FILES = ["index.sqlite", "index.sqlite-wal", "index.sqlite-shm"];
291
+
292
+ const defaultQuarantineCorruptIndex: QuarantineCorruptIndex = (qmdDir, tsSuffix) => {
293
+ let movedBase: string | null = null;
294
+ for (const name of CORRUPTIBLE_DB_FILES) {
295
+ const src = path.join(qmdDir, name);
296
+ try {
297
+ if (!fs.existsSync(src)) continue;
298
+ const dest = `${src}.corrupt-${tsSuffix}`;
299
+ fs.renameSync(src, dest);
300
+ if (name === "index.sqlite") movedBase = dest;
301
+ } catch {
302
+ // Best-effort: move what we can. A file we can't move is left in place;
303
+ // the corruption guard still prevents further writes this cycle.
304
+ }
305
+ }
306
+ return movedBase;
307
+ };
308
+
309
+ /** Filesystem-safe timestamp suffix for a quarantined DB file. */
310
+ function quarantineSuffix(nowMs: number): string {
311
+ return new Date(nowMs).toISOString().replace(/[:.]/g, "-");
312
+ }
313
+
48
314
  const DEFAULT_CHANGED_PATH_DEBOUNCE_MS = 60_000;
49
315
 
50
316
  export interface ReindexOptions {
@@ -71,6 +337,10 @@ export interface ReindexOptions {
71
337
  readCompanies?: (companiesDir: string) => string[];
72
338
  /** Returns true if the knowledge dir has at least one indexable .md file. */
73
339
  hasIndexableMarkdown?: (knowledgeDir: string) => boolean;
340
+ /** Reindex-lock acquirer override for tests. */
341
+ acquireReindexLock?: AcquireReindexLock;
342
+ /** Corrupt-index quarantine override for tests. */
343
+ quarantineCorruptIndex?: QuarantineCorruptIndex;
74
344
  /** Optional diagnostic sink for unexpected swallowed failures. */
75
345
  log?: (diagnostic: {
76
346
  event: string;
@@ -80,6 +350,20 @@ export interface ReindexOptions {
80
350
  }) => void;
81
351
  }
82
352
 
353
+ export interface ReindexResult {
354
+ qmdAvailable: boolean;
355
+ collectionsAdded: string[];
356
+ updated: boolean;
357
+ embedded: boolean;
358
+ pendingDirty: boolean;
359
+ /** True when the cycle was skipped because another writer held the lock. */
360
+ lockBusy: boolean;
361
+ /** True when a qmd command exceeded the exec timeout and was aborted. */
362
+ timedOut: boolean;
363
+ /** True when a corrupt index was detected and quarantined this cycle. */
364
+ corruptionQuarantined: boolean;
365
+ }
366
+
83
367
  /**
84
368
  * Reindex qmd for an HQ tree after a sync. Never throws — all failures are
85
369
  * swallowed so a reindex problem can never mask or fail the sync result.
@@ -89,21 +373,20 @@ export interface ReindexOptions {
89
373
  export function reindexAfterSync(
90
374
  hqRoot: string,
91
375
  opts: ReindexOptions = {},
92
- ): {
93
- qmdAvailable: boolean;
94
- collectionsAdded: string[];
95
- updated: boolean;
96
- embedded: boolean;
97
- pendingDirty: boolean;
98
- } {
376
+ ): ReindexResult {
99
377
  const exec = opts.exec ?? defaultExec;
100
378
  const existsSync = opts.existsSync ?? fs.existsSync;
101
- const result = {
379
+ const acquireLock = opts.acquireReindexLock ?? defaultAcquireReindexLock;
380
+ const quarantine = opts.quarantineCorruptIndex ?? defaultQuarantineCorruptIndex;
381
+ const result: ReindexResult = {
102
382
  qmdAvailable: false,
103
- collectionsAdded: [] as string[],
383
+ collectionsAdded: [],
104
384
  updated: false,
105
385
  embedded: false,
106
386
  pendingDirty: false,
387
+ lockBusy: false,
388
+ timedOut: false,
389
+ corruptionQuarantined: false,
107
390
  };
108
391
 
109
392
  try {
@@ -125,75 +408,109 @@ export function reindexAfterSync(
125
408
  return result;
126
409
  }
127
410
 
128
- // Guard: qmd must be installed. `qmd collection list` doubles as the
129
- // availability probe AND the source for which collections already exist.
130
- const list = exec(["collection", "list"]);
131
- if (list.status !== 0) return result; // qmd absent or errored — no-op
132
- result.qmdAvailable = true;
133
- const existingCollections = list.stdout;
134
-
135
- // 1. Auto-register missing company knowledge collections.
136
- if (registrationMayBeStale) {
137
- const companiesDir = path.join(hqRoot, "companies");
138
- const slugs = (opts.readCompanies ?? defaultReadCompanies)(companiesDir);
139
- for (const slug of slugs) {
140
- const knowledgeDir = path.join(companiesDir, slug, "knowledge");
141
- if (!existsSync(knowledgeDir)) continue;
142
- const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
143
- if (!hasMd) continue;
144
- // Already registered? qmd collection URIs look like `qmd://<slug>/`.
145
- if (existingCollections.includes(`qmd://${slug}/`)) continue;
146
-
147
- const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
148
- if (add.status === 0) {
149
- exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
150
- result.collectionsAdded.push(slug);
151
- }
152
- }
411
+ const nowMs = opts.nowMs ?? Date.now();
412
+ const pendingSinceMs =
413
+ pendingDirty && typeof state.pendingSinceMs === "number"
414
+ ? state.pendingSinceMs
415
+ : nowMs;
416
+
417
+ // Serialize qmd writers. If another writer (this runner from a prior cycle,
418
+ // or an interactive session's qmd maintenance) holds the lock, SKIP this
419
+ // cycle and mark the tree dirty so the next sync retries — never write onto
420
+ // the vector store concurrently (feedback_b9a369ff).
421
+ const lockPath = path.join(hqRoot, ".qmd", ".reindex.lock");
422
+ const lock = acquireLock(lockPath);
423
+ if (!lock) {
424
+ writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
425
+ result.lockBusy = true;
426
+ result.pendingDirty = true;
427
+ return result;
153
428
  }
154
429
 
155
- // 2. Incremental lexical reindex.
156
- const shouldUpdate = dirtyFromChanges || pendingDirty;
157
- if (shouldUpdate) {
158
- const nowMs = opts.nowMs ?? Date.now();
159
- const debounceMs =
160
- opts.debounceMs ??
161
- (changedPaths === undefined ? 0 : DEFAULT_CHANGED_PATH_DEBOUNCE_MS);
162
- const pendingSinceMs =
163
- pendingDirty && typeof state.pendingSinceMs === "number"
164
- ? state.pendingSinceMs
165
- : nowMs;
166
- if (debounceMs > 0 && nowMs - pendingSinceMs < debounceMs) {
167
- writeState(statePath, {
168
- ...state,
169
- pendingDirty: true,
170
- pendingSinceMs,
171
- });
172
- result.pendingDirty = true;
430
+ try {
431
+ // Guard: qmd must be installed. `qmd collection list` doubles as the
432
+ // availability probe AND the source for which collections already exist.
433
+ const list = exec(["collection", "list"]);
434
+ if (list.status !== 0) return result; // qmd absent or errored — no-op
435
+ // An already-corrupt store can surface here even on the lexical side.
436
+ if (looksCorrupt(list)) {
437
+ handleCorruption(hqRoot, statePath, state, pendingSinceMs, nowMs, quarantine, result);
173
438
  return result;
174
439
  }
440
+ result.qmdAvailable = true;
441
+ const existingCollections = list.stdout;
442
+
443
+ // 1. Auto-register missing company knowledge collections.
444
+ if (registrationMayBeStale) {
445
+ const companiesDir = path.join(hqRoot, "companies");
446
+ const slugs = (opts.readCompanies ?? defaultReadCompanies)(companiesDir);
447
+ for (const slug of slugs) {
448
+ const knowledgeDir = path.join(companiesDir, slug, "knowledge");
449
+ if (!existsSync(knowledgeDir)) continue;
450
+ const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
451
+ if (!hasMd) continue;
452
+ // Already registered? qmd collection URIs look like `qmd://<slug>/`.
453
+ if (existingCollections.includes(`qmd://${slug}/`)) continue;
175
454
 
176
- const update = exec(["update"]);
177
- result.updated = update.status === 0;
178
- if (result.updated) {
179
- writeState(statePath, {
180
- pendingDirty: false,
181
- lastSuccessMs: nowMs,
182
- });
183
- } else {
184
- writeState(statePath, {
185
- ...state,
186
- pendingDirty: true,
187
- pendingSinceMs,
188
- });
189
- result.pendingDirty = true;
455
+ const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
456
+ if (add.status === 0) {
457
+ exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
458
+ result.collectionsAdded.push(slug);
459
+ }
460
+ }
190
461
  }
191
- }
192
462
 
193
- // 3. Embeddings only on explicit request.
194
- if (opts.embed) {
195
- const embed = exec(["embed"]);
196
- result.embedded = embed.status === 0;
463
+ // 2. Incremental lexical reindex.
464
+ const shouldUpdate = dirtyFromChanges || pendingDirty;
465
+ if (shouldUpdate) {
466
+ const debounceMs =
467
+ opts.debounceMs ??
468
+ (changedPaths === undefined ? 0 : DEFAULT_CHANGED_PATH_DEBOUNCE_MS);
469
+ if (debounceMs > 0 && nowMs - pendingSinceMs < debounceMs) {
470
+ writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
471
+ result.pendingDirty = true;
472
+ return result;
473
+ }
474
+
475
+ const update = exec(["update"]);
476
+ if (looksCorrupt(update)) {
477
+ handleCorruption(hqRoot, statePath, state, pendingSinceMs, nowMs, quarantine, result);
478
+ return result;
479
+ }
480
+ if (update.timedOut) {
481
+ // Aborted mid-pass by the timeout. Do NOT treat as done and do NOT
482
+ // proceed to embed — mark dirty and retry cleanly next cycle.
483
+ writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
484
+ result.timedOut = true;
485
+ result.pendingDirty = true;
486
+ return result;
487
+ }
488
+ result.updated = update.status === 0;
489
+ if (result.updated) {
490
+ writeState(statePath, { pendingDirty: false, lastSuccessMs: nowMs });
491
+ } else {
492
+ writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
493
+ result.pendingDirty = true;
494
+ }
495
+ }
496
+
497
+ // 3. Embeddings only on explicit request.
498
+ if (opts.embed) {
499
+ const embed = exec(["embed"]);
500
+ if (looksCorrupt(embed)) {
501
+ handleCorruption(hqRoot, statePath, state, pendingSinceMs, nowMs, quarantine, result);
502
+ return result;
503
+ }
504
+ if (embed.timedOut) {
505
+ writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
506
+ result.timedOut = true;
507
+ result.pendingDirty = true;
508
+ return result;
509
+ }
510
+ result.embedded = embed.status === 0;
511
+ }
512
+ } finally {
513
+ lock.release();
197
514
  }
198
515
  } catch (err) {
199
516
  try {
@@ -212,6 +529,27 @@ export function reindexAfterSync(
212
529
  return result;
213
530
  }
214
531
 
532
+ /**
533
+ * Quarantine the corrupt DB aside and mark the tree dirty so the NEXT cycle
534
+ * rebuilds from scratch under the lock, instead of writing further onto an
535
+ * already-damaged vector store. Mutates `result` in place.
536
+ */
537
+ function handleCorruption(
538
+ hqRoot: string,
539
+ statePath: string,
540
+ state: QmdReindexState,
541
+ pendingSinceMs: number,
542
+ nowMs: number,
543
+ quarantine: QuarantineCorruptIndex,
544
+ result: ReindexResult,
545
+ ): void {
546
+ const qmdDir = path.join(hqRoot, ".qmd");
547
+ quarantine(qmdDir, quarantineSuffix(nowMs));
548
+ writeState(statePath, { ...state, pendingDirty: true, pendingSinceMs });
549
+ result.corruptionQuarantined = true;
550
+ result.pendingDirty = true;
551
+ }
552
+
215
553
  interface QmdReindexState {
216
554
  pendingDirty?: boolean;
217
555
  pendingSinceMs?: number;
package/src/s3.test.ts CHANGED
@@ -95,6 +95,7 @@ import {
95
95
  FILE_BTIME_META_KEY,
96
96
  classifyVaultKey,
97
97
  validateVaultUploadKey,
98
+ createStagedSymlink,
98
99
  replaceStagedPath,
99
100
  sweepStaleStagedFiles,
100
101
  } from "./s3.js";
@@ -733,6 +734,84 @@ describe("uploadSymlink", () => {
733
734
  });
734
735
  });
735
736
 
737
+ describe("createStagedSymlink", () => {
738
+ it("uses an absolute junction for a Windows directory without requiring symlink privilege", () => {
739
+ const linkPath = path.join(path.sep, "vault", ".agents", "skills");
740
+ const target = "../.claude/skills";
741
+ const absTarget = path.resolve(path.dirname(linkPath), target);
742
+ const symlink = vi.fn(
743
+ (_target: string, _linkPath: string, type?: fs.symlink.Type) => {
744
+ if (type !== "junction") {
745
+ throw Object.assign(new Error("operation not permitted, symlink"), {
746
+ code: "EPERM",
747
+ });
748
+ }
749
+ },
750
+ );
751
+
752
+ expect(() =>
753
+ createStagedSymlink(target, linkPath, {
754
+ platform: "win32",
755
+ symlink,
756
+ statIsDirectory: () => true,
757
+ }),
758
+ ).not.toThrow();
759
+ expect(symlink).toHaveBeenCalledWith(absTarget, linkPath, "junction");
760
+ expect(path.isAbsolute(symlink.mock.calls[0]![0])).toBe(true);
761
+ });
762
+
763
+ it("resolves a relative Windows overlay target before creating a junction", () => {
764
+ const linkPath = path.join(path.sep, "vault", ".codex", "claude");
765
+ const target = "../.claude/skills";
766
+ const symlink = vi.fn();
767
+
768
+ createStagedSymlink(target, linkPath, {
769
+ platform: "win32",
770
+ symlink,
771
+ statIsDirectory: () => undefined,
772
+ });
773
+
774
+ expect(symlink).toHaveBeenCalledWith(
775
+ path.resolve(path.dirname(linkPath), target),
776
+ linkPath,
777
+ "junction",
778
+ );
779
+ });
780
+
781
+ it("preserves a relative target and omits the type outside Windows", () => {
782
+ const linkPath = path.join(path.sep, "vault", ".agents", "skills");
783
+ const target = "../.claude/skills";
784
+ const symlink = vi.fn();
785
+ const statIsDirectory = vi.fn(() => true);
786
+
787
+ createStagedSymlink(target, linkPath, {
788
+ platform: "linux",
789
+ symlink,
790
+ statIsDirectory,
791
+ });
792
+
793
+ expect(symlink).toHaveBeenCalledWith(target, linkPath);
794
+ expect(statIsDirectory).not.toHaveBeenCalled();
795
+ });
796
+
797
+ it("uses a genuine Windows file symlink for a regular-file target", () => {
798
+ const linkPath = path.join(path.sep, "vault", "config-link.json");
799
+ const target = "../shared/config.json";
800
+ const absTarget = path.resolve(path.dirname(linkPath), target);
801
+ const symlink = vi.fn();
802
+ const statIsDirectory = vi.fn(() => false);
803
+
804
+ createStagedSymlink(target, linkPath, {
805
+ platform: "win32",
806
+ symlink,
807
+ statIsDirectory,
808
+ });
809
+
810
+ expect(statIsDirectory).toHaveBeenCalledWith(absTarget);
811
+ expect(symlink).toHaveBeenCalledWith(target, linkPath, "file");
812
+ });
813
+ });
814
+
736
815
  describe("downloadFile", () => {
737
816
  let tmpRoot: string;
738
817