@gmickel/gno 2.9.1 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.9.1.zip → gno-browser-clipper-v2.10.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.10.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +3 -2
- package/spec/cli.md +73 -2
- package/spec/output-schemas/collection-list.schema.json +5 -0
- package/src/app/bun-runtime.ts +41 -0
- package/src/cli/commands/daemon.ts +12 -0
- package/src/cli/commands/doctor.ts +61 -0
- package/src/cli/commands/mcp/config-editors.ts +56 -8
- package/src/cli/commands/mcp/install.ts +159 -24
- package/src/cli/commands/mcp/launch-path.ts +87 -0
- package/src/cli/commands/mcp/paths.ts +30 -6
- package/src/cli/commands/mcp/repair-entry.ts +160 -0
- package/src/cli/commands/mcp/status.ts +55 -15
- package/src/cli/commands/mcp.ts +4 -0
- package/src/cli/commands/serve.ts +3 -0
- package/src/cli/commands/status.ts +24 -0
- package/src/cli/detach.ts +52 -5
- package/src/cli/program.ts +19 -5
- package/src/cli/runtime-health.ts +328 -0
- package/src/config/types.ts +3 -0
- package/src/core/network-boundary-inventory.ts +16 -0
- package/src/core/project-profile-apply-state.ts +3 -0
- package/src/core/resident-shutdown-handler.ts +23 -0
- package/src/core/sighup-disposition.ts +66 -0
- package/src/core/time-slice.ts +34 -0
- package/src/ingestion/file-processor.ts +118 -17
- package/src/ingestion/graph-reconciliation.ts +18 -4
- package/src/ingestion/prepare-worker.ts +20 -2
- package/src/ingestion/sync.ts +70 -4
- package/src/serve/parent-lifeline.ts +44 -0
- package/src/serve/public/components/BootstrapStatus.tsx +3 -3
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/resident-runtime.ts +24 -1
- package/src/serve/resident-shutdown.ts +4 -0
- package/src/serve/server.ts +22 -1
- package/src/serve/status.ts +2 -1
- package/src/serve/watch-reconciliation.ts +134 -1
- package/src/serve/watch-service-flush-generation.ts +77 -3
- package/src/serve/watch-service.ts +27 -3
- package/src/sessions/import-child.ts +79 -11
- package/src/store/migrations/035-vector-length-index.ts +30 -0
- package/src/store/migrations/index.ts +2 -0
- package/src/store/sqlite/adapter.ts +4 -1
- package/src/store/vector/freshness.ts +13 -6
- package/browser-extension/artifacts/gno-browser-clipper-v2.9.1.zip.sha256 +0 -1
|
@@ -209,6 +209,7 @@ export type ResidentRuntimeDeps = {
|
|
|
209
209
|
callbacks?: CollectionWatchCallbacks;
|
|
210
210
|
syncOptions?: Parameters<typeof withContentTypeRules>[0];
|
|
211
211
|
acquireWriteLease?: () => Promise<(() => Promise<void>) | null>;
|
|
212
|
+
reconcileOnStart?: boolean;
|
|
212
213
|
}) => CollectionWatchService;
|
|
213
214
|
modelManagerFactory?: (config: Config) => ModelManager;
|
|
214
215
|
};
|
|
@@ -392,6 +393,8 @@ export async function startResidentRuntime(
|
|
|
392
393
|
},
|
|
393
394
|
syncOptions: withContentTypeRules({}, initialConfig),
|
|
394
395
|
acquireWriteLease: leaseFor(`gno ${mode} (watch sync)`),
|
|
396
|
+
// serve has no initial sync of its own; the daemon runs one unless told not to.
|
|
397
|
+
reconcileOnStart: mode === "serve",
|
|
395
398
|
});
|
|
396
399
|
watchService.start();
|
|
397
400
|
// A backlog left by `--no-embed` or an earlier run gets a pass without
|
|
@@ -502,8 +505,24 @@ export async function startResidentRuntime(
|
|
|
502
505
|
indexName: canonicalizeIndexName(options.index ?? DEFAULT_INDEX_NAME),
|
|
503
506
|
};
|
|
504
507
|
let configRefresh: Promise<void> | undefined;
|
|
508
|
+
// Every read request refreshes the config. Re-reading and re-parsing an
|
|
509
|
+
// unchanged file each time kept the main thread busy under load; skip it
|
|
510
|
+
// while the file's identity, size and times match the last good load. Any
|
|
511
|
+
// write, rename, chmod or delete changes the stamp, so a broken file is
|
|
512
|
+
// still reported, never served stale.
|
|
513
|
+
let loadedConfigStamp: string | undefined;
|
|
514
|
+
const configStamp = async (): Promise<string | undefined> => {
|
|
515
|
+
try {
|
|
516
|
+
const stat = await Bun.file(instance.configPath).stat();
|
|
517
|
+
return `${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`;
|
|
518
|
+
} catch {
|
|
519
|
+
return undefined;
|
|
520
|
+
}
|
|
521
|
+
};
|
|
505
522
|
const refreshConfig = (): Promise<void> => {
|
|
506
523
|
configRefresh ??= (async () => {
|
|
524
|
+
const stamp = await configStamp();
|
|
525
|
+
if (stamp !== undefined && stamp === loadedConfigStamp) return;
|
|
507
526
|
const file = await readInstanceConfig(
|
|
508
527
|
instance,
|
|
509
528
|
deps.loadConfig ?? loadConfig
|
|
@@ -514,7 +533,10 @@ export async function startResidentRuntime(
|
|
|
514
533
|
collections: file.collections,
|
|
515
534
|
contexts: file.contexts,
|
|
516
535
|
};
|
|
517
|
-
if (Bun.deepEquals(next, served))
|
|
536
|
+
if (Bun.deepEquals(next, served)) {
|
|
537
|
+
loadedConfigStamp = stamp;
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
518
540
|
await adoptServedConfig(
|
|
519
541
|
{
|
|
520
542
|
...instance,
|
|
@@ -529,6 +551,7 @@ export async function startResidentRuntime(
|
|
|
529
551
|
},
|
|
530
552
|
next
|
|
531
553
|
);
|
|
554
|
+
loadedConfigStamp = stamp;
|
|
532
555
|
})().finally(() => {
|
|
533
556
|
configRefresh = undefined;
|
|
534
557
|
});
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
SHUTDOWN_ABORT_MS,
|
|
17
17
|
SHUTDOWN_EXIT_MS,
|
|
18
18
|
} from "../core/shutdown-budget";
|
|
19
|
+
import { killImportChildren } from "../sessions/import-child";
|
|
19
20
|
|
|
20
21
|
interface ResidentShutdownResources {
|
|
21
22
|
options: ResidentRuntimeOptions;
|
|
@@ -78,6 +79,9 @@ export async function disposeResidentResources(
|
|
|
78
79
|
(error: unknown) => console.error("Resident cleanup failed:", error)
|
|
79
80
|
);
|
|
80
81
|
stopFindings();
|
|
82
|
+
// A session import runs in a child that would outlive this process and hold
|
|
83
|
+
// up the drain below; cutting it short is safe (one transaction per file).
|
|
84
|
+
killImportChildren();
|
|
81
85
|
const schedulerDrain = observe(() =>
|
|
82
86
|
scheduler.stop ? scheduler.stop() : scheduler.dispose()
|
|
83
87
|
);
|
package/src/serve/server.ts
CHANGED
|
@@ -3,6 +3,8 @@ import type { RequestPeerServer } from "./request-locality";
|
|
|
3
3
|
import type { ResidentRuntime } from "./resident-runtime";
|
|
4
4
|
import type { ContextHolder } from "./routes/api";
|
|
5
5
|
|
|
6
|
+
import { markResidentShutdownHandlerInstalled } from "../core/resident-shutdown-handler";
|
|
7
|
+
import { sighupHandlerAllowed } from "../core/sighup-disposition";
|
|
6
8
|
import { getActivePreset } from "../llm/registry";
|
|
7
9
|
import {
|
|
8
10
|
isHttpGatewayLoopbackBind,
|
|
@@ -25,6 +27,7 @@ import {
|
|
|
25
27
|
* @module src/serve/server
|
|
26
28
|
*/
|
|
27
29
|
import { withRemoteHostPathRedaction } from "./host-path-redaction";
|
|
30
|
+
import { watchParentLifeline } from "./parent-lifeline";
|
|
28
31
|
import { PDFJS_ASSET_CACHE_CONTROL } from "./pdfjs-assets";
|
|
29
32
|
// HTML import - Bun handles bundling TSX/CSS automatically via routes
|
|
30
33
|
import homepage from "./public/index.html";
|
|
@@ -401,7 +404,7 @@ export function loopbackHttpOrigin(host: string, port: number): string {
|
|
|
401
404
|
|
|
402
405
|
/**
|
|
403
406
|
* Start the web server.
|
|
404
|
-
* Opens DB once, closes on SIGINT/SIGTERM.
|
|
407
|
+
* Opens DB once, closes on SIGINT/SIGTERM/SIGHUP.
|
|
405
408
|
*/
|
|
406
409
|
export async function startServer(
|
|
407
410
|
options: ServeOptions = {},
|
|
@@ -478,9 +481,22 @@ export async function startServer(
|
|
|
478
481
|
|
|
479
482
|
process.once("SIGINT", shutdown);
|
|
480
483
|
process.once("SIGTERM", shutdown);
|
|
484
|
+
// A closed terminal or logout: shut down cleanly, as on SIGTERM, unless
|
|
485
|
+
// SIGHUP was ignored at start (nohup), which must keep working.
|
|
486
|
+
const handleSighup = sighupHandlerAllowed();
|
|
487
|
+
if (handleSighup) process.once("SIGHUP", shutdown);
|
|
488
|
+
markResidentShutdownHandlerInstalled();
|
|
489
|
+
// A launcher that owns this process (the desktop shell) closes our stdin
|
|
490
|
+
// when it exits or is killed; shut down instead of running orphaned.
|
|
491
|
+
const stopLifeline = watchParentLifeline(() => {
|
|
492
|
+
console.log("\nParent process gone.");
|
|
493
|
+
shutdown();
|
|
494
|
+
});
|
|
481
495
|
const removeShutdownHandlers = (): void => {
|
|
482
496
|
process.off("SIGINT", shutdown);
|
|
483
497
|
process.off("SIGTERM", shutdown);
|
|
498
|
+
if (handleSighup) process.off("SIGHUP", shutdown);
|
|
499
|
+
stopLifeline();
|
|
484
500
|
};
|
|
485
501
|
|
|
486
502
|
// Start server with try/catch for port-in-use etc.
|
|
@@ -1778,6 +1794,11 @@ export async function startServer(
|
|
|
1778
1794
|
await dependencies.waitForShutdown(shutdownController.signal);
|
|
1779
1795
|
} else {
|
|
1780
1796
|
await new Promise<void>((resolve) => {
|
|
1797
|
+
// The lifeline can fire while the listener is still starting.
|
|
1798
|
+
if (shutdownController.signal.aborted) {
|
|
1799
|
+
resolve();
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1781
1802
|
shutdownController.signal.addEventListener("abort", () => resolve(), {
|
|
1782
1803
|
once: true,
|
|
1783
1804
|
});
|
package/src/serve/status.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
SuggestedCollection,
|
|
13
13
|
} from "./status-model";
|
|
14
14
|
|
|
15
|
+
import { BUN_ENGINE_RANGE } from "../app/bun-runtime";
|
|
15
16
|
import { getModelsCachePath } from "../app/constants";
|
|
16
17
|
import { buildContentTypeBoostStatus } from "../config/content-types";
|
|
17
18
|
import { isEmptyActivationCollection } from "../core/activation-empty";
|
|
@@ -513,7 +514,7 @@ async function buildBootstrapState(
|
|
|
513
514
|
kind: "bun",
|
|
514
515
|
strategy: "manual-install-beta",
|
|
515
516
|
currentVersion: Bun.version,
|
|
516
|
-
requiredVersion:
|
|
517
|
+
requiredVersion: BUN_ENGINE_RANGE,
|
|
517
518
|
ready: true,
|
|
518
519
|
managedByApp: false,
|
|
519
520
|
summary: `This beta runs on Bun ${Bun.version}.`,
|
|
@@ -10,8 +10,13 @@
|
|
|
10
10
|
import type { Collection } from "../config/types";
|
|
11
11
|
import type { SqliteAdapter } from "../store/sqlite/adapter";
|
|
12
12
|
|
|
13
|
+
import { relPathUnderAnyPrefix } from "../ingestion";
|
|
13
14
|
import { WATCHER_ACTIVE_SOURCE_PATH_MAX } from "../store/types";
|
|
14
15
|
import { fallbackClassifyDirtyHints } from "./watch-reconciliation-fallback";
|
|
16
|
+
import {
|
|
17
|
+
fallbackFs,
|
|
18
|
+
inspectNoFollowPresence,
|
|
19
|
+
} from "./watch-reconciliation-fallback-disk";
|
|
15
20
|
import {
|
|
16
21
|
filterEligiblePaths,
|
|
17
22
|
type ClassificationResult,
|
|
@@ -48,6 +53,112 @@ export {
|
|
|
48
53
|
type WatcherEventClassification,
|
|
49
54
|
} from "./watch-reconciliation-shared";
|
|
50
55
|
|
|
56
|
+
type StoreOnlyRemovals =
|
|
57
|
+
| { status: "ok"; paths: string[] }
|
|
58
|
+
| { status: "overflow" }
|
|
59
|
+
| { status: "error"; cause: unknown; stage: "scan" | "store" };
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Store-known active paths under the dirty hints that are gone from disk.
|
|
63
|
+
* The snapshot only knows what it scanned: a note indexed from its own exact
|
|
64
|
+
* event after the baseline is in the store but not the snapshot, and when
|
|
65
|
+
* its directory is then moved or replaced (one directory event) the snapshot
|
|
66
|
+
* diff alone would leave it active (fn-209).
|
|
67
|
+
*
|
|
68
|
+
* A path the fresh snapshot lists as a file or symlink is present. Paths in
|
|
69
|
+
* subtrees the snapshot left unproven are left alone. Every other path gets
|
|
70
|
+
* the fallback classifier's no-follow, availability-aware proof: missing or
|
|
71
|
+
* no longer a regular source is a removal; an inconclusive check is a scan
|
|
72
|
+
* error, so the watcher keeps the work and retries rather than infer a
|
|
73
|
+
* delete. The root hint (a watcher event without a filename) uses the
|
|
74
|
+
* bounded whole-collection inventory.
|
|
75
|
+
*/
|
|
76
|
+
async function storeOnlyRemovals(options: {
|
|
77
|
+
store: SqliteAdapter;
|
|
78
|
+
collection: Collection;
|
|
79
|
+
rootAbs: string;
|
|
80
|
+
dirtyHints: readonly string[];
|
|
81
|
+
next: WatcherSnapshot | null;
|
|
82
|
+
sourcePathMax: number;
|
|
83
|
+
snapshotOptions?: WatcherSnapshotOptions;
|
|
84
|
+
}): Promise<StoreOnlyRemovals> {
|
|
85
|
+
const { store, collection, rootAbs, next, sourcePathMax } = options;
|
|
86
|
+
const hints = [...new Set(options.dirtyHints)].sort();
|
|
87
|
+
const storePaths: string[] = [];
|
|
88
|
+
if (hints.includes("")) {
|
|
89
|
+
if (typeof store.listActiveSourcePaths !== "function") {
|
|
90
|
+
return { status: "ok", paths: [] };
|
|
91
|
+
}
|
|
92
|
+
const listed = await store.listActiveSourcePaths(
|
|
93
|
+
collection.name,
|
|
94
|
+
sourcePathMax
|
|
95
|
+
);
|
|
96
|
+
if (!listed.ok) {
|
|
97
|
+
if (listed.error.code === "OVERFLOW") return { status: "overflow" };
|
|
98
|
+
return {
|
|
99
|
+
status: "error",
|
|
100
|
+
cause: new Error(listed.error.message),
|
|
101
|
+
stage: "store",
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
storePaths.push(...listed.value);
|
|
105
|
+
} else {
|
|
106
|
+
if (typeof store.listActiveDescendantSourcePaths !== "function") {
|
|
107
|
+
return { status: "ok", paths: [] };
|
|
108
|
+
}
|
|
109
|
+
// A hint under another hint is covered by the outer descendant lookup.
|
|
110
|
+
const roots = hints.filter(
|
|
111
|
+
(hint, index) =>
|
|
112
|
+
!hints.slice(0, index).some((outer) => hint.startsWith(`${outer}/`))
|
|
113
|
+
);
|
|
114
|
+
for (const dir of roots) {
|
|
115
|
+
const listed = await store.listActiveDescendantSourcePaths(
|
|
116
|
+
collection.name,
|
|
117
|
+
dir,
|
|
118
|
+
Math.max(1, sourcePathMax - storePaths.length)
|
|
119
|
+
);
|
|
120
|
+
if (!listed.ok) {
|
|
121
|
+
if (listed.error.code === "OVERFLOW") return { status: "overflow" };
|
|
122
|
+
// Not a directory path the store can scope (e.g. escapes the root).
|
|
123
|
+
if (listed.error.code === "INVALID_INPUT") continue;
|
|
124
|
+
return {
|
|
125
|
+
status: "error",
|
|
126
|
+
cause: new Error(listed.error.message),
|
|
127
|
+
stage: "store",
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
storePaths.push(...listed.value);
|
|
131
|
+
if (storePaths.length > sourcePathMax) return { status: "overflow" };
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const unproven = [...(next?.unprovenSubtrees ?? [])];
|
|
136
|
+
const fs = options.snapshotOptions?.fs ?? fallbackFs();
|
|
137
|
+
const removals: string[] = [];
|
|
138
|
+
for (const path of storePaths) {
|
|
139
|
+
if (relPathUnderAnyPrefix(path, unproven)) continue;
|
|
140
|
+
const slash = path.lastIndexOf("/");
|
|
141
|
+
const parent = slash === -1 ? "" : path.slice(0, slash);
|
|
142
|
+
const kind = next?.directories
|
|
143
|
+
.get(parent)
|
|
144
|
+
?.get(path.slice(slash + 1))?.kind;
|
|
145
|
+
if (kind === "file" || kind === "symlink") continue;
|
|
146
|
+
const presence = await inspectNoFollowPresence(
|
|
147
|
+
rootAbs,
|
|
148
|
+
path,
|
|
149
|
+
fs,
|
|
150
|
+
options.snapshotOptions?.directoryAvailability
|
|
151
|
+
);
|
|
152
|
+
if (presence.status === "error") {
|
|
153
|
+
return { status: "error", cause: presence.cause, stage: "scan" };
|
|
154
|
+
}
|
|
155
|
+
if (presence.status === "missing" || !presence.indexable) {
|
|
156
|
+
removals.push(path);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return { status: "ok", paths: removals };
|
|
160
|
+
}
|
|
161
|
+
|
|
51
162
|
/**
|
|
52
163
|
* Snapshot-first classification of dirty hints. On overflow/scan/metadata
|
|
53
164
|
* failure, uses bounded store + disk enumeration without inferring deletes
|
|
@@ -96,10 +207,32 @@ export async function classifyDirtyHints(options: {
|
|
|
96
207
|
snapshotOptions
|
|
97
208
|
);
|
|
98
209
|
if (diff.status === "ok") {
|
|
210
|
+
const storeRemovals = await storeOnlyRemovals({
|
|
211
|
+
store,
|
|
212
|
+
collection,
|
|
213
|
+
rootAbs,
|
|
214
|
+
dirtyHints,
|
|
215
|
+
next: diff.nextSnapshot,
|
|
216
|
+
sourcePathMax,
|
|
217
|
+
snapshotOptions,
|
|
218
|
+
});
|
|
219
|
+
if (storeRemovals.status === "overflow") {
|
|
220
|
+
return { status: "full_reconcile", reason: "budget_overflow" };
|
|
221
|
+
}
|
|
222
|
+
if (storeRemovals.status === "error") {
|
|
223
|
+
return {
|
|
224
|
+
status: "error",
|
|
225
|
+
cause: storeRemovals.cause,
|
|
226
|
+
stage: storeRemovals.stage,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
99
229
|
return {
|
|
100
230
|
status: "ok",
|
|
101
231
|
candidates: filterEligiblePaths(diff.candidates, collection),
|
|
102
|
-
removals: filterEligiblePaths(
|
|
232
|
+
removals: filterEligiblePaths(
|
|
233
|
+
[...new Set([...diff.removals, ...storeRemovals.paths])],
|
|
234
|
+
collection
|
|
235
|
+
),
|
|
103
236
|
nextSnapshot: diff.nextSnapshot,
|
|
104
237
|
usedFallback: false,
|
|
105
238
|
};
|
|
@@ -7,18 +7,65 @@
|
|
|
7
7
|
// node:path — Bun has no path utilities
|
|
8
8
|
import { normalize } from "node:path";
|
|
9
9
|
|
|
10
|
+
import type { CollectionSyncResult } from "../ingestion";
|
|
10
11
|
import type {
|
|
11
12
|
FlushCollectionInput,
|
|
12
13
|
FlushCollectionOutcome,
|
|
13
14
|
} from "./watch-service-flush";
|
|
14
15
|
|
|
15
|
-
import { defaultSyncService } from "../ingestion";
|
|
16
|
+
import { defaultSyncService, isSourceAvailabilitySkip } from "../ingestion";
|
|
16
17
|
import { hasFileLevelSyncError } from "./watch-reconciliation";
|
|
17
18
|
import {
|
|
18
19
|
contentChangedPaths,
|
|
19
20
|
notifyCompletedSync,
|
|
20
21
|
} from "./watch-service-flush-helpers";
|
|
21
22
|
|
|
23
|
+
/** Failure codes from the store side of a sync: transient, worth a retry. */
|
|
24
|
+
const STORE_SIDE_FAILURE_CODES = new Set(["QUERY_FAILED", "STORE_ERROR"]);
|
|
25
|
+
const SOURCE_AVAILABILITY_PREFIX = "SOURCE_AVAILABILITY_";
|
|
26
|
+
|
|
27
|
+
interface ReconcileFailures {
|
|
28
|
+
/** A collection-level failure (no path): the reconcile itself is incomplete. */
|
|
29
|
+
unscoped: boolean;
|
|
30
|
+
/** Every named failed path, from file receipts and walker errors alike. */
|
|
31
|
+
named: string[];
|
|
32
|
+
/** Named paths that failed in the store, not on the file's content. */
|
|
33
|
+
storeSide: string[];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Classify the failures of a completed full-collection sync. */
|
|
37
|
+
function classifyReconcileFailures(
|
|
38
|
+
result: CollectionSyncResult
|
|
39
|
+
): ReconcileFailures {
|
|
40
|
+
const named = new Set<string>();
|
|
41
|
+
const storeSide = new Set<string>();
|
|
42
|
+
let unscoped = false;
|
|
43
|
+
for (const file of result.files ?? []) {
|
|
44
|
+
if (file.status !== "error") continue;
|
|
45
|
+
named.add(file.relPath);
|
|
46
|
+
if (STORE_SIDE_FAILURE_CODES.has(file.errorCode ?? ""))
|
|
47
|
+
storeSide.add(file.relPath);
|
|
48
|
+
}
|
|
49
|
+
// A collection whose source availability cannot be proven fails the same
|
|
50
|
+
// way on every attempt until its config or volume changes (a config edit
|
|
51
|
+
// starts a new reconcile): record it once, like a content failure.
|
|
52
|
+
let unavailable = false;
|
|
53
|
+
// Walker failures (TOO_LARGE, PERMISSION...) appear only here.
|
|
54
|
+
for (const entry of result.errors) {
|
|
55
|
+
if (isSourceAvailabilitySkip(entry.code)) continue;
|
|
56
|
+
if (!entry.relPath) {
|
|
57
|
+
if (entry.code.startsWith(SOURCE_AVAILABILITY_PREFIX)) unavailable = true;
|
|
58
|
+
else unscoped = true;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
named.add(entry.relPath);
|
|
62
|
+
if (STORE_SIDE_FAILURE_CODES.has(entry.code)) storeSide.add(entry.relPath);
|
|
63
|
+
}
|
|
64
|
+
// A failure count with nothing named is collection-level too.
|
|
65
|
+
if (named.size === 0 && !unavailable) unscoped = true;
|
|
66
|
+
return { unscoped, named: [...named], storeSide: [...storeSide] };
|
|
67
|
+
}
|
|
68
|
+
|
|
22
69
|
/**
|
|
23
70
|
* When collection generation advanced during/before flush, run full
|
|
24
71
|
* syncCollection. Failures leave durable generation work and never advance
|
|
@@ -96,11 +143,38 @@ export async function runGenerationReconcile(
|
|
|
96
143
|
contentChangedPaths(result);
|
|
97
144
|
|
|
98
145
|
if (hasFileLevelSyncError(result)) {
|
|
99
|
-
// Notify completed sync once; keep durable generation work.
|
|
100
|
-
notifyCompletedSync(input, operationPaths, result, ownership);
|
|
101
146
|
const error = new Error(
|
|
102
147
|
"One or more paths failed during watcher generation reconcile"
|
|
103
148
|
);
|
|
149
|
+
const failures = classifyReconcileFailures(result);
|
|
150
|
+
// The walk completed: the collection is reconciled except for the
|
|
151
|
+
// named failed paths (fn-211). Requeuing the whole generation
|
|
152
|
+
// re-walked the collection every backoff step for as long as one file
|
|
153
|
+
// kept failing. Store-side failures are transient and can leave a
|
|
154
|
+
// stale document, so only those paths are retried (exact paths,
|
|
155
|
+
// failure backoff). Content failures (unreadable, corrupt, too
|
|
156
|
+
// large...) are recorded once, as `gno update` and the daemon's
|
|
157
|
+
// initial sync do; the next edit to the file retries it. Keep the
|
|
158
|
+
// whole generation when any failure is collection-level (e.g. the
|
|
159
|
+
// document inventory failed) or this flush no longer owns the
|
|
160
|
+
// collection (its requeue would be dropped).
|
|
161
|
+
if (
|
|
162
|
+
!failures.unscoped &&
|
|
163
|
+
input.getCurrentGeneration() === input.ownerGeneration
|
|
164
|
+
) {
|
|
165
|
+
input.invalidateSnapshot(stillCurrent);
|
|
166
|
+
notifyCompletedSync(input, operationPaths, result, ownership);
|
|
167
|
+
input.onSyncError(failures.named, error);
|
|
168
|
+
if (failures.storeSide.length > 0) {
|
|
169
|
+
input.requeue(failures.storeSide, []);
|
|
170
|
+
return { status: "failed", error };
|
|
171
|
+
}
|
|
172
|
+
completedGeneration = currentGeneration;
|
|
173
|
+
needsWork = input.getCurrentGeneration() !== completedGeneration;
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
// Notify completed sync once; keep durable generation work.
|
|
177
|
+
notifyCompletedSync(input, operationPaths, result, ownership);
|
|
104
178
|
input.onSyncError([], error);
|
|
105
179
|
input.requeueGeneration();
|
|
106
180
|
return { status: "failed", error };
|
|
@@ -49,7 +49,11 @@ import {
|
|
|
49
49
|
} from "./watch-service-lifecycle";
|
|
50
50
|
import { runOwnedCollectionFlush } from "./watch-service-run-flush";
|
|
51
51
|
import { beginSnapshotInit } from "./watch-service-snapshot";
|
|
52
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
emptyPending,
|
|
54
|
+
pendingHasWork,
|
|
55
|
+
type CollectionPending,
|
|
56
|
+
} from "./watch-service-state";
|
|
53
57
|
|
|
54
58
|
export interface CollectionWatchState {
|
|
55
59
|
expectedCollections: string[];
|
|
@@ -111,6 +115,12 @@ interface CollectionWatchServiceOptions {
|
|
|
111
115
|
snapshotEntryCeiling?: number;
|
|
112
116
|
/** Shared writer lease taken (no wait) around each flush's writes. */
|
|
113
117
|
acquireWriteLease?: () => Promise<(() => Promise<void>) | null>;
|
|
118
|
+
/**
|
|
119
|
+
* Queue a full reconcile of every watched collection at start, so changes
|
|
120
|
+
* made while no resident ran reach the index. Off for the daemon, whose
|
|
121
|
+
* initial sync (unless --no-sync-on-start) covers the same ground.
|
|
122
|
+
*/
|
|
123
|
+
reconcileOnStart?: boolean;
|
|
114
124
|
}
|
|
115
125
|
|
|
116
126
|
export class CollectionWatchService {
|
|
@@ -152,6 +162,7 @@ export class CollectionWatchService {
|
|
|
152
162
|
readonly #snapshotFs: WatcherSnapshotFs | undefined;
|
|
153
163
|
readonly #snapshotEntryCeiling: number | undefined;
|
|
154
164
|
readonly #acquireWriteLease: CollectionWatchServiceOptions["acquireWriteLease"];
|
|
165
|
+
readonly #reconcileOnStart: boolean;
|
|
155
166
|
#nextCollectionGeneration = 0;
|
|
156
167
|
#disposed = false;
|
|
157
168
|
#lastEventAt: string | null = null;
|
|
@@ -176,11 +187,24 @@ export class CollectionWatchService {
|
|
|
176
187
|
this.#snapshotFs = options.snapshotFs;
|
|
177
188
|
this.#snapshotEntryCeiling = options.snapshotEntryCeiling;
|
|
178
189
|
this.#acquireWriteLease = options.acquireWriteLease;
|
|
190
|
+
this.#reconcileOnStart = options.reconcileOnStart ?? false;
|
|
179
191
|
}
|
|
180
192
|
|
|
181
193
|
start(): void {
|
|
182
|
-
if (
|
|
183
|
-
|
|
194
|
+
if (this.#disposed) {
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
this.updateCollections(this.#collections);
|
|
198
|
+
if (!this.#reconcileOnStart) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
// Generation work waits for the snapshot baseline, then runs a full
|
|
202
|
+
// syncCollection through the normal flush path (write lease, failure
|
|
203
|
+
// backoff, source availability). Unwatchable collections queue nothing.
|
|
204
|
+
for (const name of this.#watchers.keys()) {
|
|
205
|
+
const pending = this.#pendingByCollection.get(name) ?? emptyPending();
|
|
206
|
+
pending.generationReconcile = true;
|
|
207
|
+
this.#pendingByCollection.set(name, pending);
|
|
184
208
|
}
|
|
185
209
|
}
|
|
186
210
|
|
|
@@ -9,8 +9,11 @@
|
|
|
9
9
|
* store connection: the archive import lock, receipts and error codes are
|
|
10
10
|
* unchanged.
|
|
11
11
|
*
|
|
12
|
-
* The child reads one request from stdin and writes one JSON result line
|
|
13
|
-
* stdout. The
|
|
12
|
+
* The child reads one request line from stdin and writes one JSON result line
|
|
13
|
+
* to stdout. The parent keeps stdin open for the child's lifetime: EOF means
|
|
14
|
+
* the parent died, and the child exits instead of importing on as an orphan.
|
|
15
|
+
* A resident kills its import children when it shuts down, and every import
|
|
16
|
+
* has a timeout. The request carries the caller's loaded config: the child opens
|
|
14
17
|
* the index without syncing any config into it and imports with exactly the
|
|
15
18
|
* config the server holds. From source the child runs this module; a
|
|
16
19
|
* compiled executable re-runs itself with `IMPORT_CHILD_ENV` set, which the
|
|
@@ -87,12 +90,24 @@ async function importInThisProcess(
|
|
|
87
90
|
}
|
|
88
91
|
}
|
|
89
92
|
|
|
93
|
+
/** Longest a single import may run before its child is killed. */
|
|
94
|
+
export const SESSION_IMPORT_TIMEOUT_MS = 30 * 60_000;
|
|
95
|
+
|
|
96
|
+
/** Import children still running in this process. */
|
|
97
|
+
const liveChildren = new Set<Bun.Subprocess>();
|
|
98
|
+
|
|
99
|
+
/** Kill every running import child (resident shutdown). */
|
|
100
|
+
export function killImportChildren(): void {
|
|
101
|
+
for (const child of liveChildren) child.kill("SIGKILL");
|
|
102
|
+
}
|
|
103
|
+
|
|
90
104
|
/**
|
|
91
105
|
* Import a registered source without blocking the caller's event loop.
|
|
92
106
|
* Throws a `SessionsError` for typed failures and a plain `Error` otherwise.
|
|
93
107
|
*/
|
|
94
108
|
export async function importInChildProcess(
|
|
95
|
-
request: ImportChildRequest
|
|
109
|
+
request: ImportChildRequest,
|
|
110
|
+
options: { timeoutMs?: number } = {}
|
|
96
111
|
): Promise<SessionImportReceipt> {
|
|
97
112
|
// A compiled executable cannot run an external TS entry: it re-runs itself.
|
|
98
113
|
// Shared check: covers the POSIX `/$bunfs/` and Windows `B:/~BUN/` roots.
|
|
@@ -100,15 +115,38 @@ export async function importInChildProcess(
|
|
|
100
115
|
const child = Bun.spawn({
|
|
101
116
|
cmd: compiled ? [process.execPath] : [process.execPath, import.meta.path],
|
|
102
117
|
env: compiled ? { ...process.env, [IMPORT_CHILD_ENV]: "1" } : process.env,
|
|
103
|
-
|
|
118
|
+
// The request line, then held open as the child's lifeline.
|
|
119
|
+
stdin: "pipe",
|
|
104
120
|
stdout: "pipe",
|
|
105
121
|
stderr: "pipe",
|
|
106
122
|
});
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
123
|
+
liveChildren.add(child);
|
|
124
|
+
const timeoutMs = options.timeoutMs ?? SESSION_IMPORT_TIMEOUT_MS;
|
|
125
|
+
let timedOut = false;
|
|
126
|
+
const timer = setTimeout(() => {
|
|
127
|
+
timedOut = true;
|
|
128
|
+
child.kill("SIGKILL");
|
|
129
|
+
}, timeoutMs);
|
|
130
|
+
let output: [string, string, number];
|
|
131
|
+
try {
|
|
132
|
+
void child.stdin.write(`${JSON.stringify(request)}\n`);
|
|
133
|
+
void child.stdin.flush();
|
|
134
|
+
output = await Promise.all([
|
|
135
|
+
new Response(child.stdout).text(),
|
|
136
|
+
new Response(child.stderr).text(),
|
|
137
|
+
child.exited,
|
|
138
|
+
]);
|
|
139
|
+
} finally {
|
|
140
|
+
clearTimeout(timer);
|
|
141
|
+
liveChildren.delete(child);
|
|
142
|
+
void Promise.resolve(child.stdin.end()).catch(() => undefined);
|
|
143
|
+
}
|
|
144
|
+
const [stdout, stderr, exitCode] = output;
|
|
145
|
+
if (timedOut) {
|
|
146
|
+
throw new Error(
|
|
147
|
+
`session import timed out after ${Math.round(timeoutMs / 1000)} s`
|
|
148
|
+
);
|
|
149
|
+
}
|
|
112
150
|
const line = stdout.trim().split("\n").at(-1) ?? "";
|
|
113
151
|
let result: ImportChildResult | null = null;
|
|
114
152
|
try {
|
|
@@ -126,15 +164,43 @@ export async function importInChildProcess(
|
|
|
126
164
|
throw new Error(result.message);
|
|
127
165
|
}
|
|
128
166
|
|
|
129
|
-
/**
|
|
167
|
+
/** Read the request line; afterwards stdin is only the lifeline. */
|
|
168
|
+
async function readRequestLine(
|
|
169
|
+
reader: ReadableStreamDefaultReader<Uint8Array>
|
|
170
|
+
): Promise<string> {
|
|
171
|
+
const decoder = new TextDecoder();
|
|
172
|
+
let text = "";
|
|
173
|
+
while (!text.includes("\n")) {
|
|
174
|
+
const chunk = await reader.read();
|
|
175
|
+
if (chunk.done) break;
|
|
176
|
+
text += decoder.decode(chunk.value, { stream: true });
|
|
177
|
+
}
|
|
178
|
+
return text.split("\n")[0] ?? "";
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Child entry: one request line on stdin, one JSON result line on stdout. */
|
|
130
182
|
export async function runImportChild(): Promise<void> {
|
|
131
183
|
// Nothing this child starts may re-enter child mode.
|
|
132
184
|
delete process.env[IMPORT_CHILD_ENV];
|
|
185
|
+
const reader = Bun.stdin.stream().getReader();
|
|
186
|
+
let finished = false;
|
|
133
187
|
let result: ImportChildResult;
|
|
134
188
|
try {
|
|
135
189
|
const request = JSON.parse(
|
|
136
|
-
await
|
|
190
|
+
await readRequestLine(reader)
|
|
137
191
|
) as ImportChildRequest;
|
|
192
|
+
// The parent holds stdin open until it has our result; EOF before that
|
|
193
|
+
// means it died. Noticed between import steps.
|
|
194
|
+
void (async () => {
|
|
195
|
+
try {
|
|
196
|
+
while (!(await reader.read()).done) {
|
|
197
|
+
// nothing is written after the request
|
|
198
|
+
}
|
|
199
|
+
} catch {
|
|
200
|
+
// a broken pipe is the same as EOF
|
|
201
|
+
}
|
|
202
|
+
if (!finished) process.exit(1);
|
|
203
|
+
})();
|
|
138
204
|
result = { ok: true, receipt: await importInThisProcess(request) };
|
|
139
205
|
} catch (error) {
|
|
140
206
|
result =
|
|
@@ -146,7 +212,9 @@ export async function runImportChild(): Promise<void> {
|
|
|
146
212
|
message: error instanceof Error ? error.message : String(error),
|
|
147
213
|
};
|
|
148
214
|
}
|
|
215
|
+
finished = true;
|
|
149
216
|
process.stdout.write(`${JSON.stringify(result)}\n`);
|
|
217
|
+
void reader.cancel().catch(() => undefined);
|
|
150
218
|
}
|
|
151
219
|
|
|
152
220
|
if (import.meta.main) await runImportChild();
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration: index stored vector byte lengths per model.
|
|
3
|
+
*
|
|
4
|
+
* Status checks the stored embedding dimensions for the active model by
|
|
5
|
+
* comparing the smallest and largest vector in its partition. Without an
|
|
6
|
+
* index that is a scan of every vector on each status build (about 0.6 ms
|
|
7
|
+
* per thousand vectors), run on the resident's main thread. With this
|
|
8
|
+
* covering index each bound is a single index seek.
|
|
9
|
+
*
|
|
10
|
+
* @module src/store/migrations/035-vector-length-index
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { Database } from "bun:sqlite";
|
|
14
|
+
|
|
15
|
+
import type { Migration } from "./runner";
|
|
16
|
+
|
|
17
|
+
export const migration: Migration = {
|
|
18
|
+
version: 35,
|
|
19
|
+
name: "vector_length_index",
|
|
20
|
+
|
|
21
|
+
up(db: Database): void {
|
|
22
|
+
db.exec(
|
|
23
|
+
"CREATE INDEX IF NOT EXISTS idx_vectors_model_bytes ON content_vectors(model, length(embedding))"
|
|
24
|
+
);
|
|
25
|
+
},
|
|
26
|
+
|
|
27
|
+
down(db: Database): void {
|
|
28
|
+
db.exec("DROP INDEX IF EXISTS idx_vectors_model_bytes");
|
|
29
|
+
},
|
|
30
|
+
};
|
|
@@ -48,6 +48,7 @@ import { migration as m031 } from "./031-runtime-independent-vectors";
|
|
|
48
48
|
import { migration as m032 } from "./032-vector-runtime-callers";
|
|
49
49
|
import { migration as m033 } from "./033-drop-documents-active-index";
|
|
50
50
|
import { migration as m034 } from "./034-collection-link-workspace";
|
|
51
|
+
import { migration as m035 } from "./035-vector-length-index";
|
|
51
52
|
|
|
52
53
|
/** All migrations in order */
|
|
53
54
|
export const migrations = [
|
|
@@ -85,4 +86,5 @@ export const migrations = [
|
|
|
85
86
|
m032,
|
|
86
87
|
m033,
|
|
87
88
|
m034,
|
|
89
|
+
m035,
|
|
88
90
|
];
|
|
@@ -979,7 +979,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
979
979
|
}
|
|
980
980
|
});
|
|
981
981
|
|
|
982
|
-
transaction
|
|
982
|
+
// Read-then-write: take the write lock up front. A deferred transaction
|
|
983
|
+
// whose snapshot goes stale (another process commits meanwhile) fails at
|
|
984
|
+
// once with "database is locked" instead of waiting out the busy timeout.
|
|
985
|
+
transaction.immediate();
|
|
983
986
|
return ok(undefined);
|
|
984
987
|
} catch (cause) {
|
|
985
988
|
return err(
|