@gmickel/gno 2.11.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +3 -2
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.11.0.zip → gno-browser-clipper-v2.12.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.12.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +8 -5
- package/spec/cli.md +100 -34
- package/spec/evals-agentic.md +3 -1
- package/spec/mcp.md +9 -3
- package/spec/output-schemas/status.schema.json +28 -0
- package/src/app/context-agent-projection.ts +10 -1
- package/src/cli/commands/agents/block.ts +2 -2
- package/src/cli/commands/agents/commands.ts +6 -5
- package/src/cli/commands/agents/engine.ts +15 -2
- package/src/cli/commands/agents/harnesses.ts +21 -14
- package/src/cli/commands/cleanup.ts +46 -6
- package/src/cli/commands/collection/remove.ts +1 -0
- package/src/cli/commands/collection/rename.ts +1 -0
- package/src/cli/commands/doctor.ts +37 -0
- package/src/cli/commands/mcp/paths.ts +2 -1
- package/src/cli/commands/setup-semantic.ts +6 -1
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/skill/paths.ts +20 -2
- package/src/cli/commands/status.ts +89 -15
- package/src/cli/program.ts +19 -3
- package/src/core/activation-probe-plan.ts +0 -1
- package/src/core/config-mutation.ts +72 -55
- package/src/core/memory-remember.ts +0 -1
- package/src/core/module-path.ts +33 -0
- package/src/core/runtime-entrypoint.ts +3 -1
- package/src/ingestion/file-processor.ts +6 -3
- package/src/ingestion/prepare-file.ts +14 -4
- package/src/ingestion/source-availability/root.ts +138 -0
- package/src/ingestion/sync.ts +5 -19
- package/src/llm/native-worker/client.ts +8 -4
- package/src/llm/native-worker/errors.ts +26 -3
- package/src/mcp/http-transport.ts +7 -2
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-descriptions-core.ts +2 -2
- package/src/mcp/tools/index.ts +30 -1
- package/src/mcp/tools/status.ts +17 -2
- package/src/pipeline/query-snippet.ts +266 -0
- package/src/pipeline/search.ts +149 -19
- package/src/sdk/client.ts +14 -2
- package/src/sdk/types.ts +6 -1
- package/src/serve/public/components/ThoroughnessSelector.tsx +1 -1
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/pages/Collections.tsx +8 -0
- package/src/serve/public/pages/Search.tsx +9 -1
- package/src/serve/resident-request.ts +8 -0
- package/src/serve/resident-runtime.ts +25 -3
- package/src/serve/spa-production-build.ts +7 -3
- package/src/serve/status-model.ts +2 -0
- package/src/serve/status.ts +29 -0
- package/src/sessions/config-refresh.ts +6 -0
- package/src/sessions/import-child.ts +4 -1
- package/src/store/sqlite/adapter.ts +159 -18
- package/src/store/types.ts +45 -7
- package/browser-extension/artifacts/gno-browser-clipper-v2.11.0.zip.sha256 +0 -1
|
@@ -81,6 +81,8 @@ interface CollectionStats {
|
|
|
81
81
|
documentCount: number;
|
|
82
82
|
chunkCount: number;
|
|
83
83
|
embeddedCount: number;
|
|
84
|
+
/** Set while the folder is missing or unreadable; documents stay indexed. */
|
|
85
|
+
rootError?: { code: string; message: string };
|
|
84
86
|
include?: string[];
|
|
85
87
|
models?: Partial<Record<"embed" | "rerank" | "expand" | "gen", string>>;
|
|
86
88
|
effectiveModels?: Record<"embed" | "rerank" | "expand" | "gen", string>;
|
|
@@ -277,6 +279,12 @@ function CollectionCard({
|
|
|
277
279
|
</Tooltip>
|
|
278
280
|
</TooltipProvider>
|
|
279
281
|
)}
|
|
282
|
+
{collection.rootError && (
|
|
283
|
+
<p className="flex items-start gap-1.5 text-amber-500 text-xs">
|
|
284
|
+
<AlertCircleIcon className="mt-0.5 size-3.5 shrink-0" />
|
|
285
|
+
<span>{collection.rootError.message}</span>
|
|
286
|
+
</p>
|
|
287
|
+
)}
|
|
280
288
|
</CardHeader>
|
|
281
289
|
|
|
282
290
|
<CardContent className="pt-2">
|
|
@@ -174,8 +174,12 @@ export default function Search({ navigate }: PageProps) {
|
|
|
174
174
|
query.trim().length > 0 && !structuredQueryState.ok
|
|
175
175
|
? structuredQueryState.error.message
|
|
176
176
|
: null;
|
|
177
|
+
// Without vectors, Fast is keyword search unless modes or intent need the
|
|
178
|
+
// hybrid endpoint. With vectors, Fast is always hybrid (no expansion,
|
|
179
|
+
// rerank or graph), the same path as CLI `--fast`.
|
|
177
180
|
const forceHybridForModes =
|
|
178
181
|
thoroughness === "fast" &&
|
|
182
|
+
!hybridAvailable &&
|
|
179
183
|
(queryModes.length > 0 ||
|
|
180
184
|
intent.trim().length > 0 ||
|
|
181
185
|
(structuredQueryState.ok &&
|
|
@@ -325,6 +329,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
325
329
|
|
|
326
330
|
const useBm25 =
|
|
327
331
|
thoroughness === "fast" &&
|
|
332
|
+
!hybridAvailable &&
|
|
328
333
|
queryModes.length === 0 &&
|
|
329
334
|
intent.trim().length === 0 &&
|
|
330
335
|
!structuredQueryState.value.usedStructuredQuerySyntax;
|
|
@@ -403,6 +408,7 @@ export default function Search({ navigate }: PageProps) {
|
|
|
403
408
|
activePreset,
|
|
404
409
|
category,
|
|
405
410
|
exclude,
|
|
411
|
+
hybridAvailable,
|
|
406
412
|
intent,
|
|
407
413
|
query,
|
|
408
414
|
queryModes,
|
|
@@ -448,7 +454,9 @@ export default function Search({ navigate }: PageProps) {
|
|
|
448
454
|
|
|
449
455
|
const thoroughnessDesc =
|
|
450
456
|
thoroughness === "fast"
|
|
451
|
-
?
|
|
457
|
+
? hybridAvailable
|
|
458
|
+
? "Hybrid, no expansion or reranking"
|
|
459
|
+
: "Keyword search (no vectors)"
|
|
452
460
|
: thoroughness === "balanced"
|
|
453
461
|
? resolveDepthPolicy({ presetId: activePreset })
|
|
454
462
|
.balancedExpansionEnabled
|
|
@@ -2,6 +2,7 @@ import type { ResidentRuntime } from "./resident-runtime";
|
|
|
2
2
|
/** Admission and bounded-reader boundary for resident REST reads. */
|
|
3
3
|
|
|
4
4
|
import { withInferenceScope } from "../llm/inference-scope";
|
|
5
|
+
import { StaleCollectionsError } from "../store/sqlite/adapter";
|
|
5
6
|
import { sessionsErrorResponse } from "./routes/sessions";
|
|
6
7
|
|
|
7
8
|
/**
|
|
@@ -16,6 +17,13 @@ export async function refreshResidentConfig(
|
|
|
16
17
|
await runtime.refreshConfig?.();
|
|
17
18
|
return null;
|
|
18
19
|
} catch (error) {
|
|
20
|
+
// Names collections only (no paths), and says how to resolve it.
|
|
21
|
+
if (error instanceof StaleCollectionsError) {
|
|
22
|
+
return Response.json(
|
|
23
|
+
{ error: { code: "STALE_COLLECTIONS", message: error.message } },
|
|
24
|
+
{ status: 409 }
|
|
25
|
+
);
|
|
26
|
+
}
|
|
19
27
|
return sessionsErrorResponse(error);
|
|
20
28
|
}
|
|
21
29
|
}
|
|
@@ -61,7 +61,11 @@ import {
|
|
|
61
61
|
readInstanceConfig,
|
|
62
62
|
} from "../sessions/config-refresh";
|
|
63
63
|
import { remoteSafeSessionsError } from "../sessions/types";
|
|
64
|
-
import {
|
|
64
|
+
import {
|
|
65
|
+
SqliteAdapter,
|
|
66
|
+
StaleCollectionsError,
|
|
67
|
+
staleCollectionsMessage,
|
|
68
|
+
} from "../store/sqlite/adapter";
|
|
65
69
|
import {
|
|
66
70
|
createServerContext,
|
|
67
71
|
type CreateServerContextOptions,
|
|
@@ -519,10 +523,23 @@ export async function startResidentRuntime(
|
|
|
519
523
|
return undefined;
|
|
520
524
|
}
|
|
521
525
|
};
|
|
526
|
+
// Another client may index a collection the served config omits after this
|
|
527
|
+
// runtime started; requests fail instead of reading it.
|
|
528
|
+
const assertServedCollections = (): void => {
|
|
529
|
+
const stale = store.staleIndexedCollections(
|
|
530
|
+
ctxHolder.config.collections.map(({ name }) => name)
|
|
531
|
+
);
|
|
532
|
+
if (stale.length > 0) {
|
|
533
|
+
throw new StaleCollectionsError(staleCollectionsMessage(stale));
|
|
534
|
+
}
|
|
535
|
+
};
|
|
522
536
|
const refreshConfig = (): Promise<void> => {
|
|
523
537
|
configRefresh ??= (async () => {
|
|
524
538
|
const stamp = await configStamp();
|
|
525
|
-
if (stamp !== undefined && stamp === loadedConfigStamp)
|
|
539
|
+
if (stamp !== undefined && stamp === loadedConfigStamp) {
|
|
540
|
+
assertServedCollections();
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
526
543
|
const file = await readInstanceConfig(
|
|
527
544
|
instance,
|
|
528
545
|
deps.loadConfig ?? loadConfig
|
|
@@ -535,6 +552,7 @@ export async function startResidentRuntime(
|
|
|
535
552
|
};
|
|
536
553
|
if (Bun.deepEquals(next, served)) {
|
|
537
554
|
loadedConfigStamp = stamp;
|
|
555
|
+
assertServedCollections();
|
|
538
556
|
return;
|
|
539
557
|
}
|
|
540
558
|
await adoptServedConfig(
|
|
@@ -816,7 +834,11 @@ export async function startResidentRuntime(
|
|
|
816
834
|
await refreshConfig();
|
|
817
835
|
lastRefreshError = undefined;
|
|
818
836
|
} catch (error) {
|
|
819
|
-
|
|
837
|
+
// The stale-collections message names collections, never paths.
|
|
838
|
+
const message =
|
|
839
|
+
error instanceof StaleCollectionsError
|
|
840
|
+
? error.message
|
|
841
|
+
: remoteSafeSessionsError(error).message;
|
|
820
842
|
if (message !== lastRefreshError)
|
|
821
843
|
console.warn(`gno ${mode}: config not reloaded: ${message}`);
|
|
822
844
|
lastRefreshError = message;
|
|
@@ -6,6 +6,8 @@ import { tmpdir } from "node:os";
|
|
|
6
6
|
// node:path — no Bun path utils.
|
|
7
7
|
import { basename, join, relative } from "node:path";
|
|
8
8
|
|
|
9
|
+
import { moduleDir, modulePath } from "../core/module-path";
|
|
10
|
+
|
|
9
11
|
export const ROOT_MOUNT_MARKER = 'getElementById("root")';
|
|
10
12
|
|
|
11
13
|
export type ProductionSpaFile = {
|
|
@@ -19,7 +21,8 @@ export type ProductionSpaAssets = {
|
|
|
19
21
|
sourceHash: string;
|
|
20
22
|
};
|
|
21
23
|
|
|
22
|
-
const productionSpaPublicDir = (): string =>
|
|
24
|
+
const productionSpaPublicDir = (): string =>
|
|
25
|
+
join(moduleDir(import.meta), "public");
|
|
23
26
|
|
|
24
27
|
export const productionSpaEntryPath = (): string =>
|
|
25
28
|
join(productionSpaPublicDir(), "index.html");
|
|
@@ -79,13 +82,14 @@ const contentTypeFor = (path: string): string => {
|
|
|
79
82
|
return "application/octet-stream";
|
|
80
83
|
};
|
|
81
84
|
|
|
82
|
-
export const isBunfsPath = (path: string): boolean => {
|
|
85
|
+
export const isBunfsPath = (path: string | undefined): boolean => {
|
|
86
|
+
if (path === undefined) return false;
|
|
83
87
|
const normalized = path.replaceAll("\\", "/");
|
|
84
88
|
return normalized.includes("/$bunfs/") || normalized.startsWith("B:/~BUN/");
|
|
85
89
|
};
|
|
86
90
|
|
|
87
91
|
export const isStandaloneExecutable = (): boolean =>
|
|
88
|
-
isBunfsPath(import.meta
|
|
92
|
+
isBunfsPath(modulePath(import.meta));
|
|
89
93
|
|
|
90
94
|
const rewriteProductionHtml = (html: string, jsEntryPath: string): string => {
|
|
91
95
|
const script = `<script type="module" src="/${basename(jsEntryPath)}"></script>`;
|
|
@@ -28,6 +28,8 @@ export interface StatusCollection {
|
|
|
28
28
|
documentCount: number;
|
|
29
29
|
chunkCount: number;
|
|
30
30
|
embeddedCount: number;
|
|
31
|
+
/** Present while the collection root is missing or unreadable (path-free). */
|
|
32
|
+
rootError?: { code: string; message: string };
|
|
31
33
|
}
|
|
32
34
|
|
|
33
35
|
export interface SuggestedCollection {
|
package/src/serve/status.ts
CHANGED
|
@@ -20,6 +20,10 @@ import {
|
|
|
20
20
|
type ActivationStatus,
|
|
21
21
|
buildActivationStatus,
|
|
22
22
|
} from "../core/activation-status";
|
|
23
|
+
import {
|
|
24
|
+
checkCollectionRoots,
|
|
25
|
+
type CollectionRootError,
|
|
26
|
+
} from "../ingestion/source-availability/root";
|
|
23
27
|
import { ModelCache } from "../llm/cache";
|
|
24
28
|
import { envIsSet, resolveDownloadPolicy } from "../llm/policy";
|
|
25
29
|
import { getActivePreset, resolveModelUri } from "../llm/registry";
|
|
@@ -263,6 +267,25 @@ function buildCollectionCheck(status: IndexStatus): HealthCheck {
|
|
|
263
267
|
};
|
|
264
268
|
}
|
|
265
269
|
|
|
270
|
+
/** Folders whose root is missing or unreadable right now; null when none. */
|
|
271
|
+
function buildFolderAccessCheck(
|
|
272
|
+
rootErrors: Record<string, CollectionRootError>
|
|
273
|
+
): HealthCheck | null {
|
|
274
|
+
const names = Object.keys(rootErrors);
|
|
275
|
+
if (names.length === 0) {
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
return {
|
|
279
|
+
id: "folder-access",
|
|
280
|
+
title: "Folder access",
|
|
281
|
+
status: "warn",
|
|
282
|
+
summary: `${names.join(", ")}: folder missing or unreadable`,
|
|
283
|
+
detail: `${names.map((name) => `${name}: ${rootErrors[name]?.message}.`).join(" ")} Reconnect the drive or restore the folder; this clears as soon as it is back. If the folder moved, change the collection's path.`,
|
|
284
|
+
actionLabel: "Manage folders",
|
|
285
|
+
actionKind: "open-collections",
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
|
|
266
289
|
function buildIndexingCheck(status: IndexStatus): HealthCheck {
|
|
267
290
|
if (status.recentErrors > 0) {
|
|
268
291
|
return {
|
|
@@ -688,6 +711,7 @@ export async function buildAppStatus(
|
|
|
688
711
|
bootstrap,
|
|
689
712
|
embedCached,
|
|
690
713
|
connectorTargets,
|
|
714
|
+
rootErrors,
|
|
691
715
|
] = await Promise.all([
|
|
692
716
|
buildModelCheck(ctx, deps),
|
|
693
717
|
buildDiskCheck(status, deps),
|
|
@@ -695,6 +719,7 @@ export async function buildAppStatus(
|
|
|
695
719
|
buildBootstrapState(ctx),
|
|
696
720
|
isModelCached(preset.embed),
|
|
697
721
|
(deps.listConnectorTargets ?? getConnectorVerificationTargets)(),
|
|
722
|
+
checkCollectionRoots(ctx.config.collections),
|
|
698
723
|
]);
|
|
699
724
|
const activation = await (deps.buildActivation ?? buildActivationStatus)(
|
|
700
725
|
ctx.store,
|
|
@@ -712,6 +737,7 @@ export async function buildAppStatus(
|
|
|
712
737
|
const backgroundCheck = buildBackgroundCheck(ctx, status);
|
|
713
738
|
const checks = [
|
|
714
739
|
buildCollectionCheck(status),
|
|
740
|
+
buildFolderAccessCheck(rootErrors),
|
|
715
741
|
buildIndexingCheck(status),
|
|
716
742
|
buildActivationCheck(activation),
|
|
717
743
|
buildConnectorActivationCheck(activation),
|
|
@@ -746,6 +772,9 @@ export async function buildAppStatus(
|
|
|
746
772
|
documentCount: collection.activeDocuments,
|
|
747
773
|
chunkCount: collection.totalChunks,
|
|
748
774
|
embeddedCount: collection.embeddedChunks,
|
|
775
|
+
...(rootErrors[collection.name]
|
|
776
|
+
? { rootError: rootErrors[collection.name] }
|
|
777
|
+
: {}),
|
|
749
778
|
})),
|
|
750
779
|
totalDocuments: status.activeDocuments,
|
|
751
780
|
totalChunks: status.totalChunks,
|
|
@@ -13,6 +13,7 @@ import type { SqliteAdapter } from "../store/sqlite/adapter";
|
|
|
13
13
|
import { getIndexDbPath } from "../app/constants";
|
|
14
14
|
import { loadConfig } from "../config";
|
|
15
15
|
import { collectionEgressPolicyEpoch } from "../core/collection-egress-policy-service";
|
|
16
|
+
import { StaleCollectionsError } from "../store/sqlite/adapter";
|
|
16
17
|
import { assertSessionBinding } from "./binding";
|
|
17
18
|
import { SessionsError } from "./types";
|
|
18
19
|
|
|
@@ -76,7 +77,12 @@ export async function adoptServedConfig(
|
|
|
76
77
|
served: ServedSessionsConfig,
|
|
77
78
|
config: Config
|
|
78
79
|
): Promise<void> {
|
|
80
|
+
// A config edit never deletes indexed collections; one it dropped fails
|
|
81
|
+
// the refresh until it is added back or `gno cleanup` deletes it.
|
|
79
82
|
const collections = await served.store.syncCollections(config.collections);
|
|
83
|
+
if (!collections.ok && collections.error.code === "STALE_COLLECTIONS") {
|
|
84
|
+
throw new StaleCollectionsError(collections.error.message);
|
|
85
|
+
}
|
|
80
86
|
if (!collections.ok) {
|
|
81
87
|
throw new Error(
|
|
82
88
|
`Config saved but collection sync failed: ${collections.error.message}`
|
|
@@ -26,6 +26,7 @@ import type { Config } from "../config/types";
|
|
|
26
26
|
import type { SessionImportReceipt } from "./types";
|
|
27
27
|
|
|
28
28
|
import { getIndexDbPath } from "../app/constants";
|
|
29
|
+
import { modulePath } from "../core/module-path";
|
|
29
30
|
import { isStandaloneExecutable } from "../serve/spa-production-build";
|
|
30
31
|
import { SqliteAdapter } from "../store/sqlite/adapter";
|
|
31
32
|
import { assertSessionBinding } from "./binding";
|
|
@@ -113,7 +114,9 @@ export async function importInChildProcess(
|
|
|
113
114
|
// Shared check: covers the POSIX `/$bunfs/` and Windows `B:/~BUN/` roots.
|
|
114
115
|
const compiled = isStandaloneExecutable();
|
|
115
116
|
const child = Bun.spawn({
|
|
116
|
-
cmd: compiled
|
|
117
|
+
cmd: compiled
|
|
118
|
+
? [process.execPath]
|
|
119
|
+
: [process.execPath, modulePath(import.meta)],
|
|
117
120
|
env: compiled ? { ...process.env, [IMPORT_CHILD_ENV]: "1" } : process.env,
|
|
118
121
|
// The request line, then held open as the child's lifeline.
|
|
119
122
|
stdin: "pipe",
|
|
@@ -103,8 +103,10 @@ import type {
|
|
|
103
103
|
SavedCapsuleReverificationState,
|
|
104
104
|
SavedCapsuleVerificationExpectation,
|
|
105
105
|
SavedCapsuleVerificationRecord,
|
|
106
|
+
StaleCollection,
|
|
106
107
|
StorePort,
|
|
107
108
|
StoreResult,
|
|
109
|
+
SyncCollectionsOptions,
|
|
108
110
|
TagCount,
|
|
109
111
|
TagRow,
|
|
110
112
|
TagSource,
|
|
@@ -585,6 +587,27 @@ function resolveBusyTimeoutMs(busyTimeoutMs?: number): number {
|
|
|
585
587
|
return busyTimeoutMs;
|
|
586
588
|
}
|
|
587
589
|
|
|
590
|
+
/** The index holds collections the config omits; the message names them. */
|
|
591
|
+
export class StaleCollectionsError extends Error {}
|
|
592
|
+
|
|
593
|
+
/** Names the collections a config omits and how to resolve it. */
|
|
594
|
+
export function staleCollectionsMessage(
|
|
595
|
+
stale: readonly StaleCollection[]
|
|
596
|
+
): string {
|
|
597
|
+
const names = stale
|
|
598
|
+
.map(
|
|
599
|
+
(c) =>
|
|
600
|
+
`${c.name} (${c.documents} ${c.documents === 1 ? "document" : "documents"})`
|
|
601
|
+
)
|
|
602
|
+
.join(", ");
|
|
603
|
+
return (
|
|
604
|
+
`This index holds collections the config does not list: ${names}. ` +
|
|
605
|
+
"Nothing was changed. Add them back to the config, open a separate index " +
|
|
606
|
+
'(--index <name>, or dbPath in the SDK), or run "gno cleanup" with this ' +
|
|
607
|
+
"config to delete their index data."
|
|
608
|
+
);
|
|
609
|
+
}
|
|
610
|
+
|
|
588
611
|
export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
589
612
|
private db: Database | null = null;
|
|
590
613
|
private dbPath = "";
|
|
@@ -898,24 +921,36 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
898
921
|
// Config Sync
|
|
899
922
|
// ─────────────────────────────────────────────────────────────────────────
|
|
900
923
|
|
|
901
|
-
async syncCollections(
|
|
924
|
+
async syncCollections(
|
|
925
|
+
collections: Collection[],
|
|
926
|
+
options: SyncCollectionsOptions = {}
|
|
927
|
+
): Promise<StoreResult<void>> {
|
|
928
|
+
let stale: StaleCollection[] = [];
|
|
902
929
|
try {
|
|
903
930
|
const db = this.ensureOpen();
|
|
904
931
|
|
|
905
932
|
const transaction = db.transaction(() => {
|
|
906
|
-
//
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
//
|
|
917
|
-
|
|
918
|
-
|
|
933
|
+
// Opening an index with a config never deletes indexed collections
|
|
934
|
+
// that config omits (their documents follow by cascade): a different
|
|
935
|
+
// config on the same index would wipe it. Only explicit removals do.
|
|
936
|
+
const { remove } = options;
|
|
937
|
+
const removed = new Set(remove === "stale" ? [] : remove);
|
|
938
|
+
const unlisted = this.queryStaleCollections(
|
|
939
|
+
db,
|
|
940
|
+
collections.map((c) => c.name)
|
|
941
|
+
).filter((c) => remove !== "stale" && !removed.has(c.name));
|
|
942
|
+
// Unlisted collections without documents stay too; they only block
|
|
943
|
+
// the sync when they hold indexed documents.
|
|
944
|
+
stale = unlisted.filter((c) => c.documents > 0);
|
|
945
|
+
if (stale.length > 0 && !options.keepStale) {
|
|
946
|
+
throw new StaleCollectionsError();
|
|
947
|
+
}
|
|
948
|
+
const keep = new Set(unlisted.map((c) => c.name));
|
|
949
|
+
for (const { name } of this.queryStaleCollections(
|
|
950
|
+
db,
|
|
951
|
+
collections.map((c) => c.name)
|
|
952
|
+
)) {
|
|
953
|
+
if (!keep.has(name)) {
|
|
919
954
|
db.run("DELETE FROM collections WHERE name = ?", [name]);
|
|
920
955
|
}
|
|
921
956
|
}
|
|
@@ -985,6 +1020,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
985
1020
|
transaction.immediate();
|
|
986
1021
|
return ok(undefined);
|
|
987
1022
|
} catch (cause) {
|
|
1023
|
+
if (cause instanceof StaleCollectionsError) {
|
|
1024
|
+
return err("STALE_COLLECTIONS", staleCollectionsMessage(stale));
|
|
1025
|
+
}
|
|
988
1026
|
return err(
|
|
989
1027
|
"QUERY_FAILED",
|
|
990
1028
|
cause instanceof Error ? cause.message : "Failed to sync collections",
|
|
@@ -993,6 +1031,55 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
993
1031
|
}
|
|
994
1032
|
}
|
|
995
1033
|
|
|
1034
|
+
async listStaleCollections(
|
|
1035
|
+
configured: readonly string[]
|
|
1036
|
+
): Promise<StoreResult<StaleCollection[]>> {
|
|
1037
|
+
try {
|
|
1038
|
+
return ok(this.queryStaleCollections(this.ensureOpen(), configured));
|
|
1039
|
+
} catch (cause) {
|
|
1040
|
+
return err(
|
|
1041
|
+
"QUERY_FAILED",
|
|
1042
|
+
cause instanceof Error ? cause.message : "Failed to list collections",
|
|
1043
|
+
cause
|
|
1044
|
+
);
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/**
|
|
1049
|
+
* Indexed collections `configured` omits, checked cheaply on each read of a
|
|
1050
|
+
* long-lived client: another client may add a collection after it opened.
|
|
1051
|
+
*/
|
|
1052
|
+
staleIndexedCollections(configured: readonly string[]): StaleCollection[] {
|
|
1053
|
+
const db = this.ensureOpen();
|
|
1054
|
+
const listed = new Set(configured);
|
|
1055
|
+
const indexed = db
|
|
1056
|
+
.query<{ name: string }, []>(
|
|
1057
|
+
`SELECT c.name AS name FROM collections c
|
|
1058
|
+
WHERE EXISTS (SELECT 1 FROM documents d WHERE d.collection = c.name)`
|
|
1059
|
+
)
|
|
1060
|
+
.all();
|
|
1061
|
+
return indexed.some(({ name }) => !listed.has(name))
|
|
1062
|
+
? this.queryStaleCollections(db, configured).filter(
|
|
1063
|
+
(c) => c.documents > 0
|
|
1064
|
+
)
|
|
1065
|
+
: [];
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
private queryStaleCollections(
|
|
1069
|
+
db: Database,
|
|
1070
|
+
configured: readonly string[]
|
|
1071
|
+
): StaleCollection[] {
|
|
1072
|
+
const listed = new Set(configured);
|
|
1073
|
+
return db
|
|
1074
|
+
.query<StaleCollection, []>(
|
|
1075
|
+
`SELECT c.name AS name,
|
|
1076
|
+
(SELECT COUNT(*) FROM documents d WHERE d.collection = c.name) AS documents
|
|
1077
|
+
FROM collections c ORDER BY c.name`
|
|
1078
|
+
)
|
|
1079
|
+
.all()
|
|
1080
|
+
.filter((c) => !listed.has(c.name));
|
|
1081
|
+
}
|
|
1082
|
+
|
|
996
1083
|
async upsertCollections(
|
|
997
1084
|
collections: Collection[]
|
|
998
1085
|
): Promise<StoreResult<void>> {
|
|
@@ -3119,6 +3206,64 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
3119
3206
|
}
|
|
3120
3207
|
}
|
|
3121
3208
|
|
|
3209
|
+
async getChunkTermPresence(
|
|
3210
|
+
mirrorHashes: string[],
|
|
3211
|
+
terms: string[]
|
|
3212
|
+
): Promise<
|
|
3213
|
+
StoreResult<{ mirrorHash: string; seq: number; present: boolean[] }[]>
|
|
3214
|
+
> {
|
|
3215
|
+
try {
|
|
3216
|
+
const uniqueHashes = uniqueNonEmptyValues(mirrorHashes);
|
|
3217
|
+
if (uniqueHashes.length === 0 || terms.length === 0) return ok([]);
|
|
3218
|
+
const db = this.ensureOpen();
|
|
3219
|
+
// SQLite lower() folds ASCII only, so a non-ASCII term is also looked
|
|
3220
|
+
// up in its upper-case and capitalized spellings (ÜBERBLICK, Überblick).
|
|
3221
|
+
const spellings = terms.flatMap((term) => [
|
|
3222
|
+
term,
|
|
3223
|
+
term.toUpperCase(),
|
|
3224
|
+
`${term.charAt(0).toUpperCase()}${term.slice(1)}`,
|
|
3225
|
+
]);
|
|
3226
|
+
const columns = terms
|
|
3227
|
+
.map(
|
|
3228
|
+
() =>
|
|
3229
|
+
"(instr(lower(text), ?) > 0 OR instr(text, ?) > 0 OR instr(text, ?) > 0)"
|
|
3230
|
+
)
|
|
3231
|
+
.join(", ");
|
|
3232
|
+
const batchSize = Math.max(
|
|
3233
|
+
1,
|
|
3234
|
+
SQLITE_SAFE_PARAMETER_BATCH_SIZE - spellings.length
|
|
3235
|
+
);
|
|
3236
|
+
const rows: { mirrorHash: string; seq: number; present: boolean[] }[] =
|
|
3237
|
+
[];
|
|
3238
|
+
for (let i = 0; i < uniqueHashes.length; i += batchSize) {
|
|
3239
|
+
const batch = uniqueHashes.slice(i, i + batchSize);
|
|
3240
|
+
const values = db
|
|
3241
|
+
.query<unknown, string[]>(
|
|
3242
|
+
`SELECT mirror_hash, seq, ${columns} FROM content_chunks
|
|
3243
|
+
WHERE mirror_hash IN (${batch.map(() => "?").join(",")})
|
|
3244
|
+
ORDER BY mirror_hash, seq`
|
|
3245
|
+
)
|
|
3246
|
+
.values(...spellings, ...batch) as unknown[][];
|
|
3247
|
+
for (const [mirrorHash, seq, ...present] of values) {
|
|
3248
|
+
rows.push({
|
|
3249
|
+
mirrorHash: String(mirrorHash),
|
|
3250
|
+
seq: Number(seq),
|
|
3251
|
+
present: present.map((flag) => flag === 1),
|
|
3252
|
+
});
|
|
3253
|
+
}
|
|
3254
|
+
}
|
|
3255
|
+
return ok(rows);
|
|
3256
|
+
} catch (cause) {
|
|
3257
|
+
return err(
|
|
3258
|
+
"QUERY_FAILED",
|
|
3259
|
+
cause instanceof Error
|
|
3260
|
+
? cause.message
|
|
3261
|
+
: "Failed to read chunk term presence",
|
|
3262
|
+
cause
|
|
3263
|
+
);
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
3266
|
+
|
|
3122
3267
|
// ─────────────────────────────────────────────────────────────────────────
|
|
3123
3268
|
// FTS Search
|
|
3124
3269
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -3143,7 +3288,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
3143
3288
|
WITH fts_matches AS (
|
|
3144
3289
|
SELECT
|
|
3145
3290
|
rowid,
|
|
3146
|
-
${options.snippet ? "snippet(documents_fts, 2, '<mark>', '</mark>', '...', 32) as snippet," : ""}
|
|
3147
3291
|
bm25(
|
|
3148
3292
|
documents_fts,
|
|
3149
3293
|
${FTS5_FIELD_WEIGHTS.filepath},
|
|
@@ -3163,7 +3307,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
3163
3307
|
d.mirror_hash,
|
|
3164
3308
|
${options.chunkLanguage ? "(SELECT min(lc.seq) FROM content_chunks lc WHERE lc.mirror_hash = d.mirror_hash AND lc.language = ?) as seq," : "0 as seq,"}
|
|
3165
3309
|
fm.score as score,
|
|
3166
|
-
${options.snippet ? "fm.snippet as snippet," : ""}
|
|
3167
3310
|
d.docid,
|
|
3168
3311
|
d.uri,
|
|
3169
3312
|
d.title,
|
|
@@ -3197,7 +3340,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
3197
3340
|
mirror_hash: string;
|
|
3198
3341
|
seq: number;
|
|
3199
3342
|
score: number;
|
|
3200
|
-
snippet?: string;
|
|
3201
3343
|
docid: string;
|
|
3202
3344
|
uri: string;
|
|
3203
3345
|
title: string | null;
|
|
@@ -3247,7 +3389,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
3247
3389
|
mirrorHash: r.mirror_hash,
|
|
3248
3390
|
seq: r.seq,
|
|
3249
3391
|
score: r.score, // Raw bm25() - smaller (more negative) is better
|
|
3250
|
-
snippet: r.snippet,
|
|
3251
3392
|
docid: r.docid,
|
|
3252
3393
|
uri: r.uri,
|
|
3253
3394
|
title: r.title ?? undefined,
|
package/src/store/types.ts
CHANGED
|
@@ -68,7 +68,23 @@ export type StoreErrorCode =
|
|
|
68
68
|
| "VEC_REBUILD_FAILED"
|
|
69
69
|
| "VEC_SYNC_FAILED"
|
|
70
70
|
/** Embedding would build a separate vector partition without confirmation. */
|
|
71
|
-
| "VECTOR_PARTITION_FORK"
|
|
71
|
+
| "VECTOR_PARTITION_FORK"
|
|
72
|
+
/** The index holds indexed collections the opening config does not list. */
|
|
73
|
+
| "STALE_COLLECTIONS";
|
|
74
|
+
|
|
75
|
+
/** How a config sync treats collections the config no longer lists. */
|
|
76
|
+
export interface SyncCollectionsOptions {
|
|
77
|
+
/** Explicitly removed names, or every unlisted collection ("stale"). */
|
|
78
|
+
remove?: readonly string[] | "stale";
|
|
79
|
+
/** Leave unlisted collections untouched instead of failing. */
|
|
80
|
+
keepStale?: boolean;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** A collection in the index that the opening config does not list. */
|
|
84
|
+
export interface StaleCollection {
|
|
85
|
+
name: string;
|
|
86
|
+
documents: number;
|
|
87
|
+
}
|
|
72
88
|
|
|
73
89
|
/** Store error with structured details */
|
|
74
90
|
export interface StoreError {
|
|
@@ -738,8 +754,6 @@ export interface FtsSearchOptions extends DocumentEligibilityOptions {
|
|
|
738
754
|
* so runtime language filtering is not currently implemented.
|
|
739
755
|
*/
|
|
740
756
|
language?: string;
|
|
741
|
-
/** Include snippet with highlights */
|
|
742
|
-
snippet?: boolean;
|
|
743
757
|
/** Match documents containing ANY positive term instead of ALL of them. */
|
|
744
758
|
anyTerm?: boolean;
|
|
745
759
|
/**
|
|
@@ -770,7 +784,6 @@ export interface FtsResult {
|
|
|
770
784
|
mirrorHash: string;
|
|
771
785
|
seq: number;
|
|
772
786
|
score: number;
|
|
773
|
-
snippet?: string;
|
|
774
787
|
// Joined from documents table
|
|
775
788
|
docid?: string;
|
|
776
789
|
uri?: string;
|
|
@@ -1614,10 +1627,21 @@ export interface StorePort {
|
|
|
1614
1627
|
// ─────────────────────────────────────────────────────────────────────────
|
|
1615
1628
|
|
|
1616
1629
|
/**
|
|
1617
|
-
* Sync collections from config to DB.
|
|
1618
|
-
*
|
|
1630
|
+
* Sync collections from config to DB: adds new and updates existing ones.
|
|
1631
|
+
* A collection in the index but not in `collections` is deleted only when
|
|
1632
|
+
* `options.remove` names it (or is "stale"). One that holds documents
|
|
1633
|
+
* otherwise fails the sync with STALE_COLLECTIONS and changes nothing,
|
|
1634
|
+
* unless `keepStale` leaves it untouched.
|
|
1619
1635
|
*/
|
|
1620
|
-
syncCollections(
|
|
1636
|
+
syncCollections(
|
|
1637
|
+
collections: Collection[],
|
|
1638
|
+
options?: SyncCollectionsOptions
|
|
1639
|
+
): Promise<StoreResult<void>>;
|
|
1640
|
+
|
|
1641
|
+
/** Collections in the index that `configured` does not list, with counts. */
|
|
1642
|
+
listStaleCollections(
|
|
1643
|
+
configured: readonly string[]
|
|
1644
|
+
): Promise<StoreResult<StaleCollection[]>>;
|
|
1621
1645
|
|
|
1622
1646
|
/**
|
|
1623
1647
|
* Sync contexts from config to DB.
|
|
@@ -2159,6 +2183,20 @@ export interface StorePort {
|
|
|
2159
2183
|
keys: { mirrorHash: string; seq: number }[]
|
|
2160
2184
|
): Promise<StoreResult<Map<string, ChunkRow[]>>>;
|
|
2161
2185
|
|
|
2186
|
+
/**
|
|
2187
|
+
* Optional: for every chunk of the given documents, which of the given
|
|
2188
|
+
* lowercase terms occur in its text (substring match on ASCII-lowercased
|
|
2189
|
+
* text), computed in the database so chunk text is never loaded. Rows are
|
|
2190
|
+
* ordered by mirrorHash, then seq; `present[i]` answers `terms[i]`. Callers
|
|
2191
|
+
* fall back to getChunksBatch when this capability is absent.
|
|
2192
|
+
*/
|
|
2193
|
+
getChunkTermPresence?(
|
|
2194
|
+
mirrorHashes: string[],
|
|
2195
|
+
terms: string[]
|
|
2196
|
+
): Promise<
|
|
2197
|
+
StoreResult<{ mirrorHash: string; seq: number; present: boolean[] }[]>
|
|
2198
|
+
>;
|
|
2199
|
+
|
|
2162
2200
|
// ─────────────────────────────────────────────────────────────────────────
|
|
2163
2201
|
// FTS Search
|
|
2164
2202
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
589f77b3c4d48d0922fccb9598fc5e07dc32e9f56afa899998d49a735309eae5 gno-browser-clipper-v2.11.0.zip
|