@gmickel/gno 2.9.1 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +2 -2
  2. package/assets/spa-production.json.gz +0 -0
  3. package/browser-extension/artifacts/{gno-browser-clipper-v2.9.1.zip → gno-browser-clipper-v2.10.0.zip} +0 -0
  4. package/browser-extension/artifacts/gno-browser-clipper-v2.10.0.zip.sha256 +1 -0
  5. package/browser-extension/dist/manifest.json +1 -1
  6. package/package.json +3 -2
  7. package/spec/cli.md +73 -2
  8. package/spec/output-schemas/collection-list.schema.json +5 -0
  9. package/src/app/bun-runtime.ts +41 -0
  10. package/src/cli/commands/daemon.ts +12 -0
  11. package/src/cli/commands/doctor.ts +61 -0
  12. package/src/cli/commands/mcp/config-editors.ts +56 -8
  13. package/src/cli/commands/mcp/install.ts +159 -24
  14. package/src/cli/commands/mcp/launch-path.ts +87 -0
  15. package/src/cli/commands/mcp/paths.ts +30 -6
  16. package/src/cli/commands/mcp/repair-entry.ts +160 -0
  17. package/src/cli/commands/mcp/status.ts +55 -15
  18. package/src/cli/commands/mcp.ts +4 -0
  19. package/src/cli/commands/serve.ts +3 -0
  20. package/src/cli/commands/status.ts +24 -0
  21. package/src/cli/detach.ts +52 -5
  22. package/src/cli/program.ts +19 -5
  23. package/src/cli/runtime-health.ts +328 -0
  24. package/src/config/types.ts +3 -0
  25. package/src/core/network-boundary-inventory.ts +16 -0
  26. package/src/core/project-profile-apply-state.ts +3 -0
  27. package/src/core/resident-shutdown-handler.ts +23 -0
  28. package/src/core/sighup-disposition.ts +66 -0
  29. package/src/core/time-slice.ts +34 -0
  30. package/src/ingestion/file-processor.ts +118 -17
  31. package/src/ingestion/graph-reconciliation.ts +18 -4
  32. package/src/ingestion/prepare-worker.ts +20 -2
  33. package/src/ingestion/sync.ts +70 -4
  34. package/src/serve/parent-lifeline.ts +44 -0
  35. package/src/serve/public/components/BootstrapStatus.tsx +3 -3
  36. package/src/serve/public/globals.built.css +1 -1
  37. package/src/serve/resident-runtime.ts +24 -1
  38. package/src/serve/resident-shutdown.ts +4 -0
  39. package/src/serve/server.ts +22 -1
  40. package/src/serve/status.ts +2 -1
  41. package/src/serve/watch-reconciliation.ts +134 -1
  42. package/src/serve/watch-service-flush-generation.ts +77 -3
  43. package/src/serve/watch-service.ts +27 -3
  44. package/src/sessions/import-child.ts +79 -11
  45. package/src/store/migrations/035-vector-length-index.ts +30 -0
  46. package/src/store/migrations/index.ts +2 -0
  47. package/src/store/sqlite/adapter.ts +4 -1
  48. package/src/store/vector/freshness.ts +13 -6
  49. package/browser-extension/artifacts/gno-browser-clipper-v2.9.1.zip.sha256 +0 -1
@@ -209,6 +209,7 @@ export type ResidentRuntimeDeps = {
209
209
  callbacks?: CollectionWatchCallbacks;
210
210
  syncOptions?: Parameters<typeof withContentTypeRules>[0];
211
211
  acquireWriteLease?: () => Promise<(() => Promise<void>) | null>;
212
+ reconcileOnStart?: boolean;
212
213
  }) => CollectionWatchService;
213
214
  modelManagerFactory?: (config: Config) => ModelManager;
214
215
  };
@@ -392,6 +393,8 @@ export async function startResidentRuntime(
392
393
  },
393
394
  syncOptions: withContentTypeRules({}, initialConfig),
394
395
  acquireWriteLease: leaseFor(`gno ${mode} (watch sync)`),
396
+ // serve has no initial sync of its own; the daemon runs one unless told not to.
397
+ reconcileOnStart: mode === "serve",
395
398
  });
396
399
  watchService.start();
397
400
  // A backlog left by `--no-embed` or an earlier run gets a pass without
@@ -502,8 +505,24 @@ export async function startResidentRuntime(
502
505
  indexName: canonicalizeIndexName(options.index ?? DEFAULT_INDEX_NAME),
503
506
  };
504
507
  let configRefresh: Promise<void> | undefined;
508
+ // Every read request refreshes the config. Re-reading and re-parsing an
509
+ // unchanged file each time kept the main thread busy under load; skip it
510
+ // while the file's identity, size and times match the last good load. Any
511
+ // write, rename, chmod or delete changes the stamp, so a broken file is
512
+ // still reported, never served stale.
513
+ let loadedConfigStamp: string | undefined;
514
+ const configStamp = async (): Promise<string | undefined> => {
515
+ try {
516
+ const stat = await Bun.file(instance.configPath).stat();
517
+ return `${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`;
518
+ } catch {
519
+ return undefined;
520
+ }
521
+ };
505
522
  const refreshConfig = (): Promise<void> => {
506
523
  configRefresh ??= (async () => {
524
+ const stamp = await configStamp();
525
+ if (stamp !== undefined && stamp === loadedConfigStamp) return;
507
526
  const file = await readInstanceConfig(
508
527
  instance,
509
528
  deps.loadConfig ?? loadConfig
@@ -514,7 +533,10 @@ export async function startResidentRuntime(
514
533
  collections: file.collections,
515
534
  contexts: file.contexts,
516
535
  };
517
- if (Bun.deepEquals(next, served)) return;
536
+ if (Bun.deepEquals(next, served)) {
537
+ loadedConfigStamp = stamp;
538
+ return;
539
+ }
518
540
  await adoptServedConfig(
519
541
  {
520
542
  ...instance,
@@ -529,6 +551,7 @@ export async function startResidentRuntime(
529
551
  },
530
552
  next
531
553
  );
554
+ loadedConfigStamp = stamp;
532
555
  })().finally(() => {
533
556
  configRefresh = undefined;
534
557
  });
@@ -16,6 +16,7 @@ import {
16
16
  SHUTDOWN_ABORT_MS,
17
17
  SHUTDOWN_EXIT_MS,
18
18
  } from "../core/shutdown-budget";
19
+ import { killImportChildren } from "../sessions/import-child";
19
20
 
20
21
  interface ResidentShutdownResources {
21
22
  options: ResidentRuntimeOptions;
@@ -78,6 +79,9 @@ export async function disposeResidentResources(
78
79
  (error: unknown) => console.error("Resident cleanup failed:", error)
79
80
  );
80
81
  stopFindings();
82
+ // A session import runs in a child that would outlive this process and hold
83
+ // up the drain below; cutting it short is safe (one transaction per file).
84
+ killImportChildren();
81
85
  const schedulerDrain = observe(() =>
82
86
  scheduler.stop ? scheduler.stop() : scheduler.dispose()
83
87
  );
@@ -3,6 +3,8 @@ import type { RequestPeerServer } from "./request-locality";
3
3
  import type { ResidentRuntime } from "./resident-runtime";
4
4
  import type { ContextHolder } from "./routes/api";
5
5
 
6
+ import { markResidentShutdownHandlerInstalled } from "../core/resident-shutdown-handler";
7
+ import { sighupHandlerAllowed } from "../core/sighup-disposition";
6
8
  import { getActivePreset } from "../llm/registry";
7
9
  import {
8
10
  isHttpGatewayLoopbackBind,
@@ -25,6 +27,7 @@ import {
25
27
  * @module src/serve/server
26
28
  */
27
29
  import { withRemoteHostPathRedaction } from "./host-path-redaction";
30
+ import { watchParentLifeline } from "./parent-lifeline";
28
31
  import { PDFJS_ASSET_CACHE_CONTROL } from "./pdfjs-assets";
29
32
  // HTML import - Bun handles bundling TSX/CSS automatically via routes
30
33
  import homepage from "./public/index.html";
@@ -401,7 +404,7 @@ export function loopbackHttpOrigin(host: string, port: number): string {
401
404
 
402
405
  /**
403
406
  * Start the web server.
404
- * Opens DB once, closes on SIGINT/SIGTERM.
407
+ * Opens DB once, closes on SIGINT/SIGTERM/SIGHUP.
405
408
  */
406
409
  export async function startServer(
407
410
  options: ServeOptions = {},
@@ -478,9 +481,22 @@ export async function startServer(
478
481
 
479
482
  process.once("SIGINT", shutdown);
480
483
  process.once("SIGTERM", shutdown);
484
+ // A closed terminal or logout: shut down cleanly, as on SIGTERM, unless
485
+ // SIGHUP was ignored at start (nohup), which must keep working.
486
+ const handleSighup = sighupHandlerAllowed();
487
+ if (handleSighup) process.once("SIGHUP", shutdown);
488
+ markResidentShutdownHandlerInstalled();
489
+ // A launcher that owns this process (the desktop shell) closes our stdin
490
+ // when it exits or is killed; shut down instead of running orphaned.
491
+ const stopLifeline = watchParentLifeline(() => {
492
+ console.log("\nParent process gone.");
493
+ shutdown();
494
+ });
481
495
  const removeShutdownHandlers = (): void => {
482
496
  process.off("SIGINT", shutdown);
483
497
  process.off("SIGTERM", shutdown);
498
+ if (handleSighup) process.off("SIGHUP", shutdown);
499
+ stopLifeline();
484
500
  };
485
501
 
486
502
  // Start server with try/catch for port-in-use etc.
@@ -1778,6 +1794,11 @@ export async function startServer(
1778
1794
  await dependencies.waitForShutdown(shutdownController.signal);
1779
1795
  } else {
1780
1796
  await new Promise<void>((resolve) => {
1797
+ // The lifeline can fire while the listener is still starting.
1798
+ if (shutdownController.signal.aborted) {
1799
+ resolve();
1800
+ return;
1801
+ }
1781
1802
  shutdownController.signal.addEventListener("abort", () => resolve(), {
1782
1803
  once: true,
1783
1804
  });
@@ -12,6 +12,7 @@ import type {
12
12
  SuggestedCollection,
13
13
  } from "./status-model";
14
14
 
15
+ import { BUN_ENGINE_RANGE } from "../app/bun-runtime";
15
16
  import { getModelsCachePath } from "../app/constants";
16
17
  import { buildContentTypeBoostStatus } from "../config/content-types";
17
18
  import { isEmptyActivationCollection } from "../core/activation-empty";
@@ -513,7 +514,7 @@ async function buildBootstrapState(
513
514
  kind: "bun",
514
515
  strategy: "manual-install-beta",
515
516
  currentVersion: Bun.version,
516
- requiredVersion: ">=1.3.0",
517
+ requiredVersion: BUN_ENGINE_RANGE,
517
518
  ready: true,
518
519
  managedByApp: false,
519
520
  summary: `This beta runs on Bun ${Bun.version}.`,
@@ -10,8 +10,13 @@
10
10
  import type { Collection } from "../config/types";
11
11
  import type { SqliteAdapter } from "../store/sqlite/adapter";
12
12
 
13
+ import { relPathUnderAnyPrefix } from "../ingestion";
13
14
  import { WATCHER_ACTIVE_SOURCE_PATH_MAX } from "../store/types";
14
15
  import { fallbackClassifyDirtyHints } from "./watch-reconciliation-fallback";
16
+ import {
17
+ fallbackFs,
18
+ inspectNoFollowPresence,
19
+ } from "./watch-reconciliation-fallback-disk";
15
20
  import {
16
21
  filterEligiblePaths,
17
22
  type ClassificationResult,
@@ -48,6 +53,112 @@ export {
48
53
  type WatcherEventClassification,
49
54
  } from "./watch-reconciliation-shared";
50
55
 
56
+ type StoreOnlyRemovals =
57
+ | { status: "ok"; paths: string[] }
58
+ | { status: "overflow" }
59
+ | { status: "error"; cause: unknown; stage: "scan" | "store" };
60
+
61
+ /**
62
+ * Store-known active paths under the dirty hints that are gone from disk.
63
+ * The snapshot only knows what it scanned: a note indexed from its own exact
64
+ * event after the baseline is in the store but not the snapshot, and when
65
+ * its directory is then moved or replaced (one directory event) the snapshot
66
+ * diff alone would leave it active (fn-209).
67
+ *
68
+ * A path the fresh snapshot lists as a file or symlink is present. Paths in
69
+ * subtrees the snapshot left unproven are left alone. Every other path gets
70
+ * the fallback classifier's no-follow, availability-aware proof: missing or
71
+ * no longer a regular source is a removal; an inconclusive check is a scan
72
+ * error, so the watcher keeps the work and retries rather than infer a
73
+ * delete. The root hint (a watcher event without a filename) uses the
74
+ * bounded whole-collection inventory.
75
+ */
76
+ async function storeOnlyRemovals(options: {
77
+ store: SqliteAdapter;
78
+ collection: Collection;
79
+ rootAbs: string;
80
+ dirtyHints: readonly string[];
81
+ next: WatcherSnapshot | null;
82
+ sourcePathMax: number;
83
+ snapshotOptions?: WatcherSnapshotOptions;
84
+ }): Promise<StoreOnlyRemovals> {
85
+ const { store, collection, rootAbs, next, sourcePathMax } = options;
86
+ const hints = [...new Set(options.dirtyHints)].sort();
87
+ const storePaths: string[] = [];
88
+ if (hints.includes("")) {
89
+ if (typeof store.listActiveSourcePaths !== "function") {
90
+ return { status: "ok", paths: [] };
91
+ }
92
+ const listed = await store.listActiveSourcePaths(
93
+ collection.name,
94
+ sourcePathMax
95
+ );
96
+ if (!listed.ok) {
97
+ if (listed.error.code === "OVERFLOW") return { status: "overflow" };
98
+ return {
99
+ status: "error",
100
+ cause: new Error(listed.error.message),
101
+ stage: "store",
102
+ };
103
+ }
104
+ storePaths.push(...listed.value);
105
+ } else {
106
+ if (typeof store.listActiveDescendantSourcePaths !== "function") {
107
+ return { status: "ok", paths: [] };
108
+ }
109
+ // A hint under another hint is covered by the outer descendant lookup.
110
+ const roots = hints.filter(
111
+ (hint, index) =>
112
+ !hints.slice(0, index).some((outer) => hint.startsWith(`${outer}/`))
113
+ );
114
+ for (const dir of roots) {
115
+ const listed = await store.listActiveDescendantSourcePaths(
116
+ collection.name,
117
+ dir,
118
+ Math.max(1, sourcePathMax - storePaths.length)
119
+ );
120
+ if (!listed.ok) {
121
+ if (listed.error.code === "OVERFLOW") return { status: "overflow" };
122
+ // Not a directory path the store can scope (e.g. escapes the root).
123
+ if (listed.error.code === "INVALID_INPUT") continue;
124
+ return {
125
+ status: "error",
126
+ cause: new Error(listed.error.message),
127
+ stage: "store",
128
+ };
129
+ }
130
+ storePaths.push(...listed.value);
131
+ if (storePaths.length > sourcePathMax) return { status: "overflow" };
132
+ }
133
+ }
134
+
135
+ const unproven = [...(next?.unprovenSubtrees ?? [])];
136
+ const fs = options.snapshotOptions?.fs ?? fallbackFs();
137
+ const removals: string[] = [];
138
+ for (const path of storePaths) {
139
+ if (relPathUnderAnyPrefix(path, unproven)) continue;
140
+ const slash = path.lastIndexOf("/");
141
+ const parent = slash === -1 ? "" : path.slice(0, slash);
142
+ const kind = next?.directories
143
+ .get(parent)
144
+ ?.get(path.slice(slash + 1))?.kind;
145
+ if (kind === "file" || kind === "symlink") continue;
146
+ const presence = await inspectNoFollowPresence(
147
+ rootAbs,
148
+ path,
149
+ fs,
150
+ options.snapshotOptions?.directoryAvailability
151
+ );
152
+ if (presence.status === "error") {
153
+ return { status: "error", cause: presence.cause, stage: "scan" };
154
+ }
155
+ if (presence.status === "missing" || !presence.indexable) {
156
+ removals.push(path);
157
+ }
158
+ }
159
+ return { status: "ok", paths: removals };
160
+ }
161
+
51
162
  /**
52
163
  * Snapshot-first classification of dirty hints. On overflow/scan/metadata
53
164
  * failure, uses bounded store + disk enumeration without inferring deletes
@@ -96,10 +207,32 @@ export async function classifyDirtyHints(options: {
96
207
  snapshotOptions
97
208
  );
98
209
  if (diff.status === "ok") {
210
+ const storeRemovals = await storeOnlyRemovals({
211
+ store,
212
+ collection,
213
+ rootAbs,
214
+ dirtyHints,
215
+ next: diff.nextSnapshot,
216
+ sourcePathMax,
217
+ snapshotOptions,
218
+ });
219
+ if (storeRemovals.status === "overflow") {
220
+ return { status: "full_reconcile", reason: "budget_overflow" };
221
+ }
222
+ if (storeRemovals.status === "error") {
223
+ return {
224
+ status: "error",
225
+ cause: storeRemovals.cause,
226
+ stage: storeRemovals.stage,
227
+ };
228
+ }
99
229
  return {
100
230
  status: "ok",
101
231
  candidates: filterEligiblePaths(diff.candidates, collection),
102
- removals: filterEligiblePaths(diff.removals, collection),
232
+ removals: filterEligiblePaths(
233
+ [...new Set([...diff.removals, ...storeRemovals.paths])],
234
+ collection
235
+ ),
103
236
  nextSnapshot: diff.nextSnapshot,
104
237
  usedFallback: false,
105
238
  };
@@ -7,18 +7,65 @@
7
7
  // node:path — Bun has no path utilities
8
8
  import { normalize } from "node:path";
9
9
 
10
+ import type { CollectionSyncResult } from "../ingestion";
10
11
  import type {
11
12
  FlushCollectionInput,
12
13
  FlushCollectionOutcome,
13
14
  } from "./watch-service-flush";
14
15
 
15
- import { defaultSyncService } from "../ingestion";
16
+ import { defaultSyncService, isSourceAvailabilitySkip } from "../ingestion";
16
17
  import { hasFileLevelSyncError } from "./watch-reconciliation";
17
18
  import {
18
19
  contentChangedPaths,
19
20
  notifyCompletedSync,
20
21
  } from "./watch-service-flush-helpers";
21
22
 
23
+ /** Failure codes from the store side of a sync: transient, worth a retry. */
24
+ const STORE_SIDE_FAILURE_CODES = new Set(["QUERY_FAILED", "STORE_ERROR"]);
25
+ const SOURCE_AVAILABILITY_PREFIX = "SOURCE_AVAILABILITY_";
26
+
27
+ interface ReconcileFailures {
28
+ /** A collection-level failure (no path): the reconcile itself is incomplete. */
29
+ unscoped: boolean;
30
+ /** Every named failed path, from file receipts and walker errors alike. */
31
+ named: string[];
32
+ /** Named paths that failed in the store, not on the file's content. */
33
+ storeSide: string[];
34
+ }
35
+
36
+ /** Classify the failures of a completed full-collection sync. */
37
+ function classifyReconcileFailures(
38
+ result: CollectionSyncResult
39
+ ): ReconcileFailures {
40
+ const named = new Set<string>();
41
+ const storeSide = new Set<string>();
42
+ let unscoped = false;
43
+ for (const file of result.files ?? []) {
44
+ if (file.status !== "error") continue;
45
+ named.add(file.relPath);
46
+ if (STORE_SIDE_FAILURE_CODES.has(file.errorCode ?? ""))
47
+ storeSide.add(file.relPath);
48
+ }
49
+ // A collection whose source availability cannot be proven fails the same
50
+ // way on every attempt until its config or volume changes (a config edit
51
+ // starts a new reconcile): record it once, like a content failure.
52
+ let unavailable = false;
53
+ // Walker failures (TOO_LARGE, PERMISSION...) appear only here.
54
+ for (const entry of result.errors) {
55
+ if (isSourceAvailabilitySkip(entry.code)) continue;
56
+ if (!entry.relPath) {
57
+ if (entry.code.startsWith(SOURCE_AVAILABILITY_PREFIX)) unavailable = true;
58
+ else unscoped = true;
59
+ continue;
60
+ }
61
+ named.add(entry.relPath);
62
+ if (STORE_SIDE_FAILURE_CODES.has(entry.code)) storeSide.add(entry.relPath);
63
+ }
64
+ // A failure count with nothing named is collection-level too.
65
+ if (named.size === 0 && !unavailable) unscoped = true;
66
+ return { unscoped, named: [...named], storeSide: [...storeSide] };
67
+ }
68
+
22
69
  /**
23
70
  * When collection generation advanced during/before flush, run full
24
71
  * syncCollection. Failures leave durable generation work and never advance
@@ -96,11 +143,38 @@ export async function runGenerationReconcile(
96
143
  contentChangedPaths(result);
97
144
 
98
145
  if (hasFileLevelSyncError(result)) {
99
- // Notify completed sync once; keep durable generation work.
100
- notifyCompletedSync(input, operationPaths, result, ownership);
101
146
  const error = new Error(
102
147
  "One or more paths failed during watcher generation reconcile"
103
148
  );
149
+ const failures = classifyReconcileFailures(result);
150
+ // The walk completed: the collection is reconciled except for the
151
+ // named failed paths (fn-211). Requeuing the whole generation
152
+ // re-walked the collection every backoff step for as long as one file
153
+ // kept failing. Store-side failures are transient and can leave a
154
+ // stale document, so only those paths are retried (exact paths,
155
+ // failure backoff). Content failures (unreadable, corrupt, too
156
+ // large...) are recorded once, as `gno update` and the daemon's
157
+ // initial sync do; the next edit to the file retries it. Keep the
158
+ // whole generation when any failure is collection-level (e.g. the
159
+ // document inventory failed) or this flush no longer owns the
160
+ // collection (its requeue would be dropped).
161
+ if (
162
+ !failures.unscoped &&
163
+ input.getCurrentGeneration() === input.ownerGeneration
164
+ ) {
165
+ input.invalidateSnapshot(stillCurrent);
166
+ notifyCompletedSync(input, operationPaths, result, ownership);
167
+ input.onSyncError(failures.named, error);
168
+ if (failures.storeSide.length > 0) {
169
+ input.requeue(failures.storeSide, []);
170
+ return { status: "failed", error };
171
+ }
172
+ completedGeneration = currentGeneration;
173
+ needsWork = input.getCurrentGeneration() !== completedGeneration;
174
+ continue;
175
+ }
176
+ // Notify completed sync once; keep durable generation work.
177
+ notifyCompletedSync(input, operationPaths, result, ownership);
104
178
  input.onSyncError([], error);
105
179
  input.requeueGeneration();
106
180
  return { status: "failed", error };
@@ -49,7 +49,11 @@ import {
49
49
  } from "./watch-service-lifecycle";
50
50
  import { runOwnedCollectionFlush } from "./watch-service-run-flush";
51
51
  import { beginSnapshotInit } from "./watch-service-snapshot";
52
- import { pendingHasWork, type CollectionPending } from "./watch-service-state";
52
+ import {
53
+ emptyPending,
54
+ pendingHasWork,
55
+ type CollectionPending,
56
+ } from "./watch-service-state";
53
57
 
54
58
  export interface CollectionWatchState {
55
59
  expectedCollections: string[];
@@ -111,6 +115,12 @@ interface CollectionWatchServiceOptions {
111
115
  snapshotEntryCeiling?: number;
112
116
  /** Shared writer lease taken (no wait) around each flush's writes. */
113
117
  acquireWriteLease?: () => Promise<(() => Promise<void>) | null>;
118
+ /**
119
+ * Queue a full reconcile of every watched collection at start, so changes
120
+ * made while no resident ran reach the index. Off for the daemon, whose
121
+ * initial sync (unless --no-sync-on-start) covers the same ground.
122
+ */
123
+ reconcileOnStart?: boolean;
114
124
  }
115
125
 
116
126
  export class CollectionWatchService {
@@ -152,6 +162,7 @@ export class CollectionWatchService {
152
162
  readonly #snapshotFs: WatcherSnapshotFs | undefined;
153
163
  readonly #snapshotEntryCeiling: number | undefined;
154
164
  readonly #acquireWriteLease: CollectionWatchServiceOptions["acquireWriteLease"];
165
+ readonly #reconcileOnStart: boolean;
155
166
  #nextCollectionGeneration = 0;
156
167
  #disposed = false;
157
168
  #lastEventAt: string | null = null;
@@ -176,11 +187,24 @@ export class CollectionWatchService {
176
187
  this.#snapshotFs = options.snapshotFs;
177
188
  this.#snapshotEntryCeiling = options.snapshotEntryCeiling;
178
189
  this.#acquireWriteLease = options.acquireWriteLease;
190
+ this.#reconcileOnStart = options.reconcileOnStart ?? false;
179
191
  }
180
192
 
181
193
  start(): void {
182
- if (!this.#disposed) {
183
- this.updateCollections(this.#collections);
194
+ if (this.#disposed) {
195
+ return;
196
+ }
197
+ this.updateCollections(this.#collections);
198
+ if (!this.#reconcileOnStart) {
199
+ return;
200
+ }
201
+ // Generation work waits for the snapshot baseline, then runs a full
202
+ // syncCollection through the normal flush path (write lease, failure
203
+ // backoff, source availability). Unwatchable collections queue nothing.
204
+ for (const name of this.#watchers.keys()) {
205
+ const pending = this.#pendingByCollection.get(name) ?? emptyPending();
206
+ pending.generationReconcile = true;
207
+ this.#pendingByCollection.set(name, pending);
184
208
  }
185
209
  }
186
210
 
@@ -9,8 +9,11 @@
9
9
  * store connection: the archive import lock, receipts and error codes are
10
10
  * unchanged.
11
11
  *
12
- * The child reads one request from stdin and writes one JSON result line to
13
- * stdout. The request carries the caller's loaded config: the child opens
12
+ * The child reads one request line from stdin and writes one JSON result line
13
+ * to stdout. The parent keeps stdin open for the child's lifetime: EOF means
14
+ * the parent died, and the child exits instead of importing on as an orphan.
15
+ * A resident kills its import children when it shuts down, and every import
16
+ * has a timeout. The request carries the caller's loaded config: the child opens
14
17
  * the index without syncing any config into it and imports with exactly the
15
18
  * config the server holds. From source the child runs this module; a
16
19
  * compiled executable re-runs itself with `IMPORT_CHILD_ENV` set, which the
@@ -87,12 +90,24 @@ async function importInThisProcess(
87
90
  }
88
91
  }
89
92
 
93
+ /** Longest a single import may run before its child is killed. */
94
+ export const SESSION_IMPORT_TIMEOUT_MS = 30 * 60_000;
95
+
96
+ /** Import children still running in this process. */
97
+ const liveChildren = new Set<Bun.Subprocess>();
98
+
99
+ /** Kill every running import child (resident shutdown). */
100
+ export function killImportChildren(): void {
101
+ for (const child of liveChildren) child.kill("SIGKILL");
102
+ }
103
+
90
104
  /**
91
105
  * Import a registered source without blocking the caller's event loop.
92
106
  * Throws a `SessionsError` for typed failures and a plain `Error` otherwise.
93
107
  */
94
108
  export async function importInChildProcess(
95
- request: ImportChildRequest
109
+ request: ImportChildRequest,
110
+ options: { timeoutMs?: number } = {}
96
111
  ): Promise<SessionImportReceipt> {
97
112
  // A compiled executable cannot run an external TS entry: it re-runs itself.
98
113
  // Shared check: covers the POSIX `/$bunfs/` and Windows `B:/~BUN/` roots.
@@ -100,15 +115,38 @@ export async function importInChildProcess(
100
115
  const child = Bun.spawn({
101
116
  cmd: compiled ? [process.execPath] : [process.execPath, import.meta.path],
102
117
  env: compiled ? { ...process.env, [IMPORT_CHILD_ENV]: "1" } : process.env,
103
- stdin: new Blob([JSON.stringify(request)]),
118
+ // The request line, then held open as the child's lifeline.
119
+ stdin: "pipe",
104
120
  stdout: "pipe",
105
121
  stderr: "pipe",
106
122
  });
107
- const [stdout, stderr, exitCode] = await Promise.all([
108
- new Response(child.stdout).text(),
109
- new Response(child.stderr).text(),
110
- child.exited,
111
- ]);
123
+ liveChildren.add(child);
124
+ const timeoutMs = options.timeoutMs ?? SESSION_IMPORT_TIMEOUT_MS;
125
+ let timedOut = false;
126
+ const timer = setTimeout(() => {
127
+ timedOut = true;
128
+ child.kill("SIGKILL");
129
+ }, timeoutMs);
130
+ let output: [string, string, number];
131
+ try {
132
+ void child.stdin.write(`${JSON.stringify(request)}\n`);
133
+ void child.stdin.flush();
134
+ output = await Promise.all([
135
+ new Response(child.stdout).text(),
136
+ new Response(child.stderr).text(),
137
+ child.exited,
138
+ ]);
139
+ } finally {
140
+ clearTimeout(timer);
141
+ liveChildren.delete(child);
142
+ void Promise.resolve(child.stdin.end()).catch(() => undefined);
143
+ }
144
+ const [stdout, stderr, exitCode] = output;
145
+ if (timedOut) {
146
+ throw new Error(
147
+ `session import timed out after ${Math.round(timeoutMs / 1000)} s`
148
+ );
149
+ }
112
150
  const line = stdout.trim().split("\n").at(-1) ?? "";
113
151
  let result: ImportChildResult | null = null;
114
152
  try {
@@ -126,15 +164,43 @@ export async function importInChildProcess(
126
164
  throw new Error(result.message);
127
165
  }
128
166
 
129
- /** Child entry: one request on stdin, one JSON result line on stdout. */
167
+ /** Read the request line; afterwards stdin is only the lifeline. */
168
+ async function readRequestLine(
169
+ reader: ReadableStreamDefaultReader<Uint8Array>
170
+ ): Promise<string> {
171
+ const decoder = new TextDecoder();
172
+ let text = "";
173
+ while (!text.includes("\n")) {
174
+ const chunk = await reader.read();
175
+ if (chunk.done) break;
176
+ text += decoder.decode(chunk.value, { stream: true });
177
+ }
178
+ return text.split("\n")[0] ?? "";
179
+ }
180
+
181
+ /** Child entry: one request line on stdin, one JSON result line on stdout. */
130
182
  export async function runImportChild(): Promise<void> {
131
183
  // Nothing this child starts may re-enter child mode.
132
184
  delete process.env[IMPORT_CHILD_ENV];
185
+ const reader = Bun.stdin.stream().getReader();
186
+ let finished = false;
133
187
  let result: ImportChildResult;
134
188
  try {
135
189
  const request = JSON.parse(
136
- await new Response(Bun.stdin.stream()).text()
190
+ await readRequestLine(reader)
137
191
  ) as ImportChildRequest;
192
+ // The parent holds stdin open until it has our result; EOF before that
193
+ // means it died. Noticed between import steps.
194
+ void (async () => {
195
+ try {
196
+ while (!(await reader.read()).done) {
197
+ // nothing is written after the request
198
+ }
199
+ } catch {
200
+ // a broken pipe is the same as EOF
201
+ }
202
+ if (!finished) process.exit(1);
203
+ })();
138
204
  result = { ok: true, receipt: await importInThisProcess(request) };
139
205
  } catch (error) {
140
206
  result =
@@ -146,7 +212,9 @@ export async function runImportChild(): Promise<void> {
146
212
  message: error instanceof Error ? error.message : String(error),
147
213
  };
148
214
  }
215
+ finished = true;
149
216
  process.stdout.write(`${JSON.stringify(result)}\n`);
217
+ void reader.cancel().catch(() => undefined);
150
218
  }
151
219
 
152
220
  if (import.meta.main) await runImportChild();
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Migration: index stored vector byte lengths per model.
3
+ *
4
+ * Status checks the stored embedding dimensions for the active model by
5
+ * comparing the smallest and largest vector in its partition. Without an
6
+ * index that is a scan of every vector on each status build (about 0.6 ms
7
+ * per thousand vectors), run on the resident's main thread. With this
8
+ * covering index each bound is a single index seek.
9
+ *
10
+ * @module src/store/migrations/035-vector-length-index
11
+ */
12
+
13
+ import type { Database } from "bun:sqlite";
14
+
15
+ import type { Migration } from "./runner";
16
+
17
+ export const migration: Migration = {
18
+ version: 35,
19
+ name: "vector_length_index",
20
+
21
+ up(db: Database): void {
22
+ db.exec(
23
+ "CREATE INDEX IF NOT EXISTS idx_vectors_model_bytes ON content_vectors(model, length(embedding))"
24
+ );
25
+ },
26
+
27
+ down(db: Database): void {
28
+ db.exec("DROP INDEX IF EXISTS idx_vectors_model_bytes");
29
+ },
30
+ };
@@ -48,6 +48,7 @@ import { migration as m031 } from "./031-runtime-independent-vectors";
48
48
  import { migration as m032 } from "./032-vector-runtime-callers";
49
49
  import { migration as m033 } from "./033-drop-documents-active-index";
50
50
  import { migration as m034 } from "./034-collection-link-workspace";
51
+ import { migration as m035 } from "./035-vector-length-index";
51
52
 
52
53
  /** All migrations in order */
53
54
  export const migrations = [
@@ -85,4 +86,5 @@ export const migrations = [
85
86
  m032,
86
87
  m033,
87
88
  m034,
89
+ m035,
88
90
  ];
@@ -979,7 +979,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
979
979
  }
980
980
  });
981
981
 
982
- transaction();
982
+ // Read-then-write: take the write lock up front. A deferred transaction
983
+ // whose snapshot goes stale (another process commits meanwhile) fails at
984
+ // once with "database is locked" instead of waiting out the busy timeout.
985
+ transaction.immediate();
983
986
  return ok(undefined);
984
987
  } catch (cause) {
985
988
  return err(