@gmickel/gno 0.38.0 → 0.39.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -9
- package/assets/skill/SKILL.md +7 -0
- package/assets/skill/cli-reference.md +6 -0
- package/package.json +3 -1
- package/src/cli/commands/collection/clear-embeddings.ts +83 -0
- package/src/cli/commands/collection/index.ts +1 -0
- package/src/cli/commands/models/use.ts +19 -4
- package/src/cli/commands/status.ts +4 -1
- package/src/cli/program.ts +14 -0
- package/src/config/types.ts +4 -4
- package/src/mcp/tools/status.ts +4 -1
- package/src/sdk/client.ts +5 -1
- package/src/serve/public/components/AIModelSelector.tsx +8 -1
- package/src/serve/public/components/CollectionModelDialog.tsx +3 -1
- package/src/serve/public/pages/Collections.tsx +138 -0
- package/src/serve/routes/api.ts +67 -0
- package/src/serve/server.ts +20 -0
- package/src/serve/status.ts +4 -2
- package/src/store/sqlite/adapter.ts +148 -9
- package/src/store/types.ts +19 -1
- package/src/store/vector/sqlite-vec.ts +1 -1
package/README.md
CHANGED
|
@@ -87,14 +87,27 @@ gno daemon
|
|
|
87
87
|
|
|
88
88
|
## What's New
|
|
89
89
|
|
|
90
|
-
> Latest release: [v0.
|
|
90
|
+
> Latest release: [v0.39.1](./CHANGELOG.md#0391---2026-04-06)
|
|
91
91
|
> Full release history: [CHANGELOG.md](./CHANGELOG.md)
|
|
92
92
|
|
|
93
93
|
- **Retrieval Quality Upgrade**: stronger BM25 lexical handling, code-aware chunking, terminal result hyperlinks, and per-collection model overrides
|
|
94
94
|
- **Code Embedding Benchmarks**: new benchmark workflow across canonical, real-GNO, and pinned OSS slices for comparing alternate embedding models
|
|
95
|
-
- **
|
|
95
|
+
- **Default Embed Model**: built-in presets now use `Qwen3-Embedding-0.6B-GGUF` after it beat `bge-m3` on both code and multilingual prose benchmark lanes
|
|
96
96
|
- **Regression Fixes**: tightened phrase/negation/hyphen/underscore BM25 behavior, cleaned non-TTY hyperlink output, improved `gno doctor` chunking visibility, and fixed the embedding autoresearch harness
|
|
97
97
|
|
|
98
|
+
### Upgrading Existing Collections
|
|
99
|
+
|
|
100
|
+
If you already had collections indexed before the default embed-model switch to
|
|
101
|
+
`Qwen3-Embedding-0.6B-GGUF`, run:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
gno models pull --embed
|
|
105
|
+
gno embed
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
That regenerates embeddings for the new default model. Old vectors are kept
|
|
109
|
+
until you explicitly clear stale embeddings.
|
|
110
|
+
|
|
98
111
|
### Fine-Tuned Model Quick Use
|
|
99
112
|
|
|
100
113
|
```yaml
|
|
@@ -103,7 +116,7 @@ models:
|
|
|
103
116
|
presets:
|
|
104
117
|
- id: slim-tuned
|
|
105
118
|
name: GNO Slim Tuned
|
|
106
|
-
embed: hf:
|
|
119
|
+
embed: hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf
|
|
107
120
|
rerank: hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf
|
|
108
121
|
expand: hf:guiltylemon/gno-expansion-slim-retrieval-v1/gno-expansion-auto-entity-lock-default-mix-lr95-f16.gguf
|
|
109
122
|
gen: hf:unsloth/Qwen3-1.7B-GGUF/Qwen3-1.7B-Q4_K_M.gguf
|
|
@@ -659,11 +672,11 @@ graph TD
|
|
|
659
672
|
|
|
660
673
|
Models auto-download on first use to `~/.cache/gno/models/`. For deterministic startup, set `GNO_NO_AUTO_DOWNLOAD=1` and use `gno models pull` explicitly. Alternatively, offload to a GPU server on your network using HTTP backends.
|
|
661
674
|
|
|
662
|
-
| Model
|
|
663
|
-
|
|
|
664
|
-
|
|
|
665
|
-
| Qwen3-Reranker-0.6B
|
|
666
|
-
| Qwen/SmolLM
|
|
675
|
+
| Model | Purpose | Size |
|
|
676
|
+
| :------------------- | :------------------------------------ | :----------- |
|
|
677
|
+
| Qwen3-Embedding-0.6B | Embeddings (multilingual) | ~640MB |
|
|
678
|
+
| Qwen3-Reranker-0.6B | Cross-encoder reranking (32K context) | ~700MB |
|
|
679
|
+
| Qwen/SmolLM | Query expansion + AI answers | ~600MB-1.2GB |
|
|
667
680
|
|
|
668
681
|
### Model Presets
|
|
669
682
|
|
|
@@ -814,7 +827,28 @@ Why this is the current recommendation:
|
|
|
814
827
|
Trade-off:
|
|
815
828
|
|
|
816
829
|
- Qwen is slower to embed than `bge-m3`
|
|
817
|
-
-
|
|
830
|
+
- existing users upgrading to the new default may need to run `gno embed` again so vector and hybrid retrieval catch up
|
|
831
|
+
|
|
832
|
+
### General Multilingual Embedding Benchmark
|
|
833
|
+
|
|
834
|
+
GNO also now has a separate public-docs benchmark lane for normal markdown/prose
|
|
835
|
+
collections:
|
|
836
|
+
|
|
837
|
+
```bash
|
|
838
|
+
bun run bench:general-embeddings --candidate bge-m3-incumbent --write
|
|
839
|
+
bun run bench:general-embeddings --candidate qwen3-embedding-0.6b --write
|
|
840
|
+
```
|
|
841
|
+
|
|
842
|
+
Current signal on the public multilingual FastAPI-docs fixture:
|
|
843
|
+
|
|
844
|
+
- `bge-m3`: vector nDCG@10 `0.350`, hybrid nDCG@10 `0.642`
|
|
845
|
+
- `Qwen3-Embedding-0.6B-GGUF`: vector nDCG@10 `0.859`, hybrid nDCG@10 `0.947`
|
|
846
|
+
|
|
847
|
+
Interpretation:
|
|
848
|
+
|
|
849
|
+
- Qwen is now the strongest general multilingual embedding model we have tested
|
|
850
|
+
- built-in presets now use Qwen by default
|
|
851
|
+
- existing users may need to run `gno embed` again after upgrading so current collections catch up
|
|
818
852
|
|
|
819
853
|
---
|
|
820
854
|
|
package/assets/skill/SKILL.md
CHANGED
|
@@ -199,6 +199,13 @@ Good default guidance:
|
|
|
199
199
|
gno embed --collection gno-code
|
|
200
200
|
```
|
|
201
201
|
|
|
202
|
+
If you want to remove old vectors after switching:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
gno collection clear-embeddings gno-code # stale models only
|
|
206
|
+
gno collection clear-embeddings gno-code --all # remove everything, then re-embed
|
|
207
|
+
```
|
|
208
|
+
|
|
202
209
|
## Reference Documentation
|
|
203
210
|
|
|
204
211
|
| Topic | File |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gmickel/gno",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.39.1",
|
|
4
4
|
"description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"embeddings",
|
|
@@ -71,6 +71,8 @@
|
|
|
71
71
|
"eval:hybrid:delta": "bun scripts/hybrid-benchmark.ts --delta",
|
|
72
72
|
"bench:code-embeddings": "bun scripts/code-embedding-benchmark.ts",
|
|
73
73
|
"bench:code-embeddings:write": "bun scripts/code-embedding-benchmark.ts --write",
|
|
74
|
+
"bench:general-embeddings": "bun scripts/general-embedding-benchmark.ts",
|
|
75
|
+
"bench:general-embeddings:write": "bun scripts/general-embedding-benchmark.ts --write",
|
|
74
76
|
"eval:retrieval-candidates": "bun scripts/retrieval-candidate-benchmark.ts",
|
|
75
77
|
"eval:retrieval-candidates:write": "bun scripts/retrieval-candidate-benchmark.ts --write",
|
|
76
78
|
"eval:watch": "bun --bun evalite watch",
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* gno collection clear-embeddings - Remove stale or all embeddings for a collection.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { getIndexDbPath } from "../../../app/constants";
|
|
6
|
+
import { loadConfig } from "../../../config";
|
|
7
|
+
import { resolveModelUri } from "../../../llm/registry";
|
|
8
|
+
import { SqliteAdapter } from "../../../store/sqlite/adapter";
|
|
9
|
+
import { CliError } from "../../errors";
|
|
10
|
+
|
|
11
|
+
interface ClearEmbeddingsOptions {
|
|
12
|
+
all?: boolean;
|
|
13
|
+
json?: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function collectionClearEmbeddings(
|
|
17
|
+
name: string,
|
|
18
|
+
options: ClearEmbeddingsOptions = {}
|
|
19
|
+
): Promise<void> {
|
|
20
|
+
const configResult = await loadConfig();
|
|
21
|
+
if (!configResult.ok) {
|
|
22
|
+
throw new CliError(
|
|
23
|
+
"RUNTIME",
|
|
24
|
+
`Failed to load config: ${configResult.error.message}`
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const config = configResult.value;
|
|
29
|
+
const collection = config.collections.find(
|
|
30
|
+
(item) => item.name === name.toLowerCase()
|
|
31
|
+
);
|
|
32
|
+
if (!collection) {
|
|
33
|
+
throw new CliError("VALIDATION", `Collection not found: ${name}`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const store = new SqliteAdapter();
|
|
37
|
+
const openResult = await store.open(getIndexDbPath(), config.ftsTokenizer);
|
|
38
|
+
if (!openResult.ok) {
|
|
39
|
+
throw new CliError("RUNTIME", openResult.error.message);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const mode = options.all ? "all" : "stale";
|
|
44
|
+
const activeModel = resolveModelUri(
|
|
45
|
+
config,
|
|
46
|
+
"embed",
|
|
47
|
+
undefined,
|
|
48
|
+
collection.name
|
|
49
|
+
);
|
|
50
|
+
const result = await store.clearEmbeddingsForCollection(collection.name, {
|
|
51
|
+
mode,
|
|
52
|
+
activeModel,
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
if (!result.ok) {
|
|
56
|
+
throw new CliError("RUNTIME", result.error.message);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (options.json) {
|
|
60
|
+
process.stdout.write(`${JSON.stringify(result.value, null, 2)}\n`);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const lines = [
|
|
65
|
+
`Cleared ${result.value.deletedVectors} embedding(s) for ${result.value.collection}.`,
|
|
66
|
+
`Mode: ${result.value.mode}`,
|
|
67
|
+
];
|
|
68
|
+
if (result.value.deletedModels.length > 0) {
|
|
69
|
+
lines.push(`Models: ${result.value.deletedModels.join(", ")}`);
|
|
70
|
+
}
|
|
71
|
+
if (result.value.protectedSharedVectors > 0) {
|
|
72
|
+
lines.push(
|
|
73
|
+
`Retained ${result.value.protectedSharedVectors} shared vector(s) still referenced by other active collections.`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
if (mode === "all") {
|
|
77
|
+
lines.push(`Run: gno embed --collection ${result.value.collection}`);
|
|
78
|
+
}
|
|
79
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
80
|
+
} finally {
|
|
81
|
+
await store.close();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { createDefaultConfig, loadConfig } from "../../../config";
|
|
9
9
|
import { saveConfig } from "../../../config/saver";
|
|
10
|
-
import { getPreset, listPresets } from "../../../llm/registry";
|
|
10
|
+
import { getPreset, listPresets, resolveModelUri } from "../../../llm/registry";
|
|
11
11
|
|
|
12
12
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
13
13
|
// Types
|
|
@@ -19,7 +19,12 @@ export interface ModelsUseOptions {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
export type ModelsUseResult =
|
|
22
|
-
| {
|
|
22
|
+
| {
|
|
23
|
+
success: true;
|
|
24
|
+
preset: string;
|
|
25
|
+
name: string;
|
|
26
|
+
embedModelChanged: boolean;
|
|
27
|
+
}
|
|
23
28
|
| { success: false; error: string };
|
|
24
29
|
|
|
25
30
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -36,6 +41,7 @@ export async function modelsUse(
|
|
|
36
41
|
// Load existing config or create default
|
|
37
42
|
const configResult = await loadConfig(options.configPath);
|
|
38
43
|
const config = configResult.ok ? configResult.value : createDefaultConfig();
|
|
44
|
+
const previousEmbedModel = resolveModelUri(config, "embed");
|
|
39
45
|
|
|
40
46
|
// Check if preset exists
|
|
41
47
|
const preset = getPreset(config, presetId);
|
|
@@ -72,7 +78,12 @@ export async function modelsUse(
|
|
|
72
78
|
};
|
|
73
79
|
}
|
|
74
80
|
|
|
75
|
-
return {
|
|
81
|
+
return {
|
|
82
|
+
success: true,
|
|
83
|
+
preset: presetId,
|
|
84
|
+
name: preset.name,
|
|
85
|
+
embedModelChanged: previousEmbedModel !== preset.embed,
|
|
86
|
+
};
|
|
76
87
|
}
|
|
77
88
|
|
|
78
89
|
/**
|
|
@@ -82,5 +93,9 @@ export function formatModelsUse(result: ModelsUseResult): string {
|
|
|
82
93
|
if (!result.success) {
|
|
83
94
|
return `Error: ${result.error}`;
|
|
84
95
|
}
|
|
85
|
-
|
|
96
|
+
const lines = [`Switched to preset: ${result.preset} (${result.name})`];
|
|
97
|
+
if (result.embedModelChanged) {
|
|
98
|
+
lines.push("Embedding model changed. Run: gno embed");
|
|
99
|
+
}
|
|
100
|
+
return lines.join("\n");
|
|
86
101
|
}
|
|
@@ -9,6 +9,7 @@ import type { IndexStatus } from "../../store/types";
|
|
|
9
9
|
|
|
10
10
|
import { getIndexDbPath } from "../../app/constants";
|
|
11
11
|
import { getConfigPaths, isInitialized, loadConfig } from "../../config";
|
|
12
|
+
import { resolveModelUri } from "../../llm/registry";
|
|
12
13
|
import { SqliteAdapter } from "../../store/sqlite/adapter";
|
|
13
14
|
|
|
14
15
|
/**
|
|
@@ -148,7 +149,9 @@ export async function status(
|
|
|
148
149
|
}
|
|
149
150
|
|
|
150
151
|
try {
|
|
151
|
-
const statusResult = await store.getStatus(
|
|
152
|
+
const statusResult = await store.getStatus({
|
|
153
|
+
embedModel: resolveModelUri(config, "embed"),
|
|
154
|
+
});
|
|
152
155
|
if (!statusResult.ok) {
|
|
153
156
|
return { success: false, error: statusResult.error.message };
|
|
154
157
|
}
|
package/src/cli/program.ts
CHANGED
|
@@ -1310,6 +1310,20 @@ function wireManagementCommands(program: Command): void {
|
|
|
1310
1310
|
await collectionRename(oldName, newName);
|
|
1311
1311
|
});
|
|
1312
1312
|
|
|
1313
|
+
collectionCmd
|
|
1314
|
+
.command("clear-embeddings <name>")
|
|
1315
|
+
.description("Clear stale or all embeddings for a collection")
|
|
1316
|
+
.option("--all", "remove all embeddings for the collection")
|
|
1317
|
+
.option("--json", "JSON output")
|
|
1318
|
+
.action(async (name: string, cmdOpts: Record<string, unknown>) => {
|
|
1319
|
+
const { collectionClearEmbeddings } =
|
|
1320
|
+
await import("./commands/collection");
|
|
1321
|
+
await collectionClearEmbeddings(name, {
|
|
1322
|
+
all: Boolean(cmdOpts.all),
|
|
1323
|
+
json: Boolean(cmdOpts.json),
|
|
1324
|
+
});
|
|
1325
|
+
});
|
|
1326
|
+
|
|
1313
1327
|
// context subcommands
|
|
1314
1328
|
const contextCmd = program
|
|
1315
1329
|
.command("context")
|
package/src/config/types.ts
CHANGED
|
@@ -190,7 +190,7 @@ export const DEFAULT_MODEL_PRESETS: ModelPreset[] = [
|
|
|
190
190
|
{
|
|
191
191
|
id: "slim-tuned",
|
|
192
192
|
name: "GNO Slim Tuned (Default, ~1GB)",
|
|
193
|
-
embed: "hf:
|
|
193
|
+
embed: "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf",
|
|
194
194
|
rerank:
|
|
195
195
|
"hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf",
|
|
196
196
|
expand:
|
|
@@ -200,7 +200,7 @@ export const DEFAULT_MODEL_PRESETS: ModelPreset[] = [
|
|
|
200
200
|
{
|
|
201
201
|
id: "slim",
|
|
202
202
|
name: "Slim (~1GB)",
|
|
203
|
-
embed: "hf:
|
|
203
|
+
embed: "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf",
|
|
204
204
|
rerank:
|
|
205
205
|
"hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf",
|
|
206
206
|
expand: "hf:unsloth/Qwen3-1.7B-GGUF/Qwen3-1.7B-Q4_K_M.gguf",
|
|
@@ -209,7 +209,7 @@ export const DEFAULT_MODEL_PRESETS: ModelPreset[] = [
|
|
|
209
209
|
{
|
|
210
210
|
id: "balanced",
|
|
211
211
|
name: "Balanced (~2GB)",
|
|
212
|
-
embed: "hf:
|
|
212
|
+
embed: "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf",
|
|
213
213
|
rerank:
|
|
214
214
|
"hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf",
|
|
215
215
|
expand:
|
|
@@ -219,7 +219,7 @@ export const DEFAULT_MODEL_PRESETS: ModelPreset[] = [
|
|
|
219
219
|
{
|
|
220
220
|
id: "quality",
|
|
221
221
|
name: "Quality (Best Answers, ~2.5GB)",
|
|
222
|
-
embed: "hf:
|
|
222
|
+
embed: "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf",
|
|
223
223
|
rerank:
|
|
224
224
|
"hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf",
|
|
225
225
|
expand:
|
package/src/mcp/tools/status.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import type { IndexStatus } from "../../store/types";
|
|
8
8
|
import type { ToolContext } from "../server";
|
|
9
9
|
|
|
10
|
+
import { resolveModelUri } from "../../llm/registry";
|
|
10
11
|
import { runTool, type ToolResult } from "./index";
|
|
11
12
|
|
|
12
13
|
type StatusInput = Record<string, never>;
|
|
@@ -66,7 +67,9 @@ export function handleStatus(
|
|
|
66
67
|
ctx,
|
|
67
68
|
"gno_status",
|
|
68
69
|
async () => {
|
|
69
|
-
const result = await ctx.store.getStatus(
|
|
70
|
+
const result = await ctx.store.getStatus({
|
|
71
|
+
embedModel: resolveModelUri(ctx.config, "embed"),
|
|
72
|
+
});
|
|
70
73
|
if (!result.ok) {
|
|
71
74
|
throw new Error(result.error.message);
|
|
72
75
|
}
|
package/src/sdk/client.ts
CHANGED
|
@@ -621,7 +621,11 @@ class GnoClientImpl implements GnoClient {
|
|
|
621
621
|
|
|
622
622
|
async status(): Promise<IndexStatus> {
|
|
623
623
|
this.assertOpen();
|
|
624
|
-
return unwrapStore(
|
|
624
|
+
return unwrapStore(
|
|
625
|
+
await this.store.getStatus({
|
|
626
|
+
embedModel: resolveModelUri(this.config, "embed"),
|
|
627
|
+
})
|
|
628
|
+
);
|
|
625
629
|
}
|
|
626
630
|
|
|
627
631
|
async update(options: GnoUpdateOptions = {}): Promise<SyncResult> {
|
|
@@ -55,6 +55,8 @@ interface SetPresetResponse {
|
|
|
55
55
|
success: boolean;
|
|
56
56
|
activePreset: string;
|
|
57
57
|
capabilities: Capabilities;
|
|
58
|
+
embedModelChanged?: boolean;
|
|
59
|
+
note?: string;
|
|
58
60
|
}
|
|
59
61
|
|
|
60
62
|
interface DownloadProgress {
|
|
@@ -355,7 +357,12 @@ export function AIModelSelector({
|
|
|
355
357
|
checkCapabilities(data.capabilities);
|
|
356
358
|
setOpen(false);
|
|
357
359
|
const presetName = presets.find((preset) => preset.id === id)?.name ?? id;
|
|
358
|
-
setNotice(
|
|
360
|
+
setNotice(
|
|
361
|
+
data.embedModelChanged
|
|
362
|
+
? (data.note ??
|
|
363
|
+
`Switched to ${presetName}. Run embeddings again so vector results catch up.`)
|
|
364
|
+
: `Switched to ${presetName}`
|
|
365
|
+
);
|
|
359
366
|
const { data: statusData } =
|
|
360
367
|
await apiFetch<AppStatusResponse>("/api/status");
|
|
361
368
|
await syncFromStatus(statusData);
|
|
@@ -133,7 +133,9 @@ export function CollectionModelDialog({
|
|
|
133
133
|
const [error, setError] = useState<string | null>(null);
|
|
134
134
|
const [saving, setSaving] = useState(false);
|
|
135
135
|
const showCodeRecommendation =
|
|
136
|
-
collection !== null &&
|
|
136
|
+
collection !== null &&
|
|
137
|
+
collectionLooksCodeHeavy(collection) &&
|
|
138
|
+
collection.effectiveModels?.embed !== CODE_EMBED_RECOMMENDATION;
|
|
137
139
|
|
|
138
140
|
useEffect(() => {
|
|
139
141
|
if (!open || !collection) {
|
|
@@ -100,12 +100,25 @@ interface SyncResponse {
|
|
|
100
100
|
jobId: string;
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
+
interface EmbeddingCleanupResponse {
|
|
104
|
+
note?: string;
|
|
105
|
+
stats: {
|
|
106
|
+
collection: string;
|
|
107
|
+
deletedVectors: number;
|
|
108
|
+
deletedModels: string[];
|
|
109
|
+
mode: "stale" | "all";
|
|
110
|
+
protectedSharedVectors: number;
|
|
111
|
+
};
|
|
112
|
+
success: boolean;
|
|
113
|
+
}
|
|
114
|
+
|
|
103
115
|
interface CollectionsResponseItem extends CollectionModelDetails {}
|
|
104
116
|
|
|
105
117
|
interface CollectionCardProps {
|
|
106
118
|
actionsDisabled: boolean;
|
|
107
119
|
collection: CollectionStats;
|
|
108
120
|
onBrowse: () => void;
|
|
121
|
+
onEmbeddingCleanup: () => void;
|
|
109
122
|
onModelSettings: () => void;
|
|
110
123
|
onReindex: () => void;
|
|
111
124
|
onRemove: () => void;
|
|
@@ -135,6 +148,7 @@ function CollectionCard({
|
|
|
135
148
|
actionsDisabled,
|
|
136
149
|
collection,
|
|
137
150
|
onBrowse,
|
|
151
|
+
onEmbeddingCleanup,
|
|
138
152
|
onModelSettings,
|
|
139
153
|
onReindex,
|
|
140
154
|
onRemove,
|
|
@@ -192,6 +206,16 @@ function CollectionCard({
|
|
|
192
206
|
<CpuIcon className="mr-2 size-4" />
|
|
193
207
|
Model settings
|
|
194
208
|
</DropdownMenuItem>
|
|
209
|
+
<DropdownMenuItem
|
|
210
|
+
disabled={actionsDisabled}
|
|
211
|
+
onClick={(event) => {
|
|
212
|
+
event.stopPropagation();
|
|
213
|
+
onEmbeddingCleanup();
|
|
214
|
+
}}
|
|
215
|
+
>
|
|
216
|
+
<DatabaseIcon className="mr-2 size-4" />
|
|
217
|
+
Embedding cleanup
|
|
218
|
+
</DropdownMenuItem>
|
|
195
219
|
<DropdownMenuSeparator />
|
|
196
220
|
<DropdownMenuItem
|
|
197
221
|
disabled={actionsDisabled || isReindexing}
|
|
@@ -316,6 +340,14 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
316
340
|
const [removeDialog, setRemoveDialog] = useState<CollectionStats | null>(
|
|
317
341
|
null
|
|
318
342
|
);
|
|
343
|
+
const [embeddingCleanupDialog, setEmbeddingCleanupDialog] =
|
|
344
|
+
useState<CollectionStats | null>(null);
|
|
345
|
+
const [embeddingCleanupBusy, setEmbeddingCleanupBusy] = useState<
|
|
346
|
+
"stale" | "all" | null
|
|
347
|
+
>(null);
|
|
348
|
+
const [embeddingCleanupNote, setEmbeddingCleanupNote] = useState<
|
|
349
|
+
string | null
|
|
350
|
+
>(null);
|
|
319
351
|
const [removing, setRemoving] = useState(false);
|
|
320
352
|
const [addDialogOpen, setAddDialogOpen] = useState(false);
|
|
321
353
|
const [modelDialogCollection, setModelDialogCollection] =
|
|
@@ -410,6 +442,34 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
410
442
|
}
|
|
411
443
|
};
|
|
412
444
|
|
|
445
|
+
const handleEmbeddingCleanup = async (mode: "stale" | "all") => {
|
|
446
|
+
if (!embeddingCleanupDialog) return;
|
|
447
|
+
|
|
448
|
+
setEmbeddingCleanupBusy(mode);
|
|
449
|
+
setEmbeddingCleanupNote(null);
|
|
450
|
+
|
|
451
|
+
const { data, error: err } = await apiFetch<EmbeddingCleanupResponse>(
|
|
452
|
+
`/api/collections/${encodeURIComponent(embeddingCleanupDialog.name)}/embeddings/clear`,
|
|
453
|
+
{
|
|
454
|
+
method: "POST",
|
|
455
|
+
body: JSON.stringify({ mode }),
|
|
456
|
+
}
|
|
457
|
+
);
|
|
458
|
+
|
|
459
|
+
setEmbeddingCleanupBusy(null);
|
|
460
|
+
|
|
461
|
+
if (err) {
|
|
462
|
+
setEmbeddingCleanupNote(err);
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
setEmbeddingCleanupNote(
|
|
467
|
+
data?.note ??
|
|
468
|
+
`Cleared ${data?.stats.deletedVectors ?? 0} embedding(s) for ${embeddingCleanupDialog.name}.`
|
|
469
|
+
);
|
|
470
|
+
await loadCollections();
|
|
471
|
+
};
|
|
472
|
+
|
|
413
473
|
// Loading state
|
|
414
474
|
if (loading) {
|
|
415
475
|
return (
|
|
@@ -577,6 +637,10 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
577
637
|
`/browse?collection=${encodeURIComponent(collection.name)}`
|
|
578
638
|
)
|
|
579
639
|
}
|
|
640
|
+
onEmbeddingCleanup={() => {
|
|
641
|
+
setEmbeddingCleanupDialog(collection);
|
|
642
|
+
setEmbeddingCleanupNote(null);
|
|
643
|
+
}}
|
|
580
644
|
onModelSettings={() => setModelDialogCollection(collection)}
|
|
581
645
|
onReindex={() => void handleReindex(collection.name)}
|
|
582
646
|
onRemove={() => setRemoveDialog(collection)}
|
|
@@ -605,6 +669,80 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
605
669
|
open={!!modelDialogCollection}
|
|
606
670
|
/>
|
|
607
671
|
|
|
672
|
+
<Dialog
|
|
673
|
+
onOpenChange={(open) => {
|
|
674
|
+
if (!open) {
|
|
675
|
+
setEmbeddingCleanupDialog(null);
|
|
676
|
+
setEmbeddingCleanupBusy(null);
|
|
677
|
+
setEmbeddingCleanupNote(null);
|
|
678
|
+
}
|
|
679
|
+
}}
|
|
680
|
+
open={!!embeddingCleanupDialog}
|
|
681
|
+
>
|
|
682
|
+
<DialogContent>
|
|
683
|
+
<DialogHeader>
|
|
684
|
+
<DialogTitle>Embedding cleanup</DialogTitle>
|
|
685
|
+
<DialogDescription>
|
|
686
|
+
Remove stale embeddings or clear all embeddings for{" "}
|
|
687
|
+
<strong>{embeddingCleanupDialog?.name}</strong>.
|
|
688
|
+
</DialogDescription>
|
|
689
|
+
</DialogHeader>
|
|
690
|
+
<div className="space-y-3 text-sm">
|
|
691
|
+
<div className="rounded-lg border border-border/40 bg-card/70 p-3">
|
|
692
|
+
<p className="font-medium">Clear stale embeddings</p>
|
|
693
|
+
<p className="mt-1 text-muted-foreground">
|
|
694
|
+
Removes embeddings for models that are not the currently active
|
|
695
|
+
embed model for this collection.
|
|
696
|
+
</p>
|
|
697
|
+
</div>
|
|
698
|
+
<div className="rounded-lg border border-secondary/30 bg-secondary/8 p-3">
|
|
699
|
+
<p className="font-medium text-secondary">Clear all embeddings</p>
|
|
700
|
+
<p className="mt-1 text-muted-foreground">
|
|
701
|
+
Removes every embedding for this collection. You will need to
|
|
702
|
+
run embeddings again afterward.
|
|
703
|
+
</p>
|
|
704
|
+
</div>
|
|
705
|
+
{embeddingCleanupNote ? (
|
|
706
|
+
<div className="rounded-lg border border-border/40 bg-card/70 p-3 text-muted-foreground">
|
|
707
|
+
{embeddingCleanupNote}
|
|
708
|
+
</div>
|
|
709
|
+
) : null}
|
|
710
|
+
</div>
|
|
711
|
+
<DialogFooter className="gap-2 sm:gap-0">
|
|
712
|
+
<Button
|
|
713
|
+
onClick={() => {
|
|
714
|
+
setEmbeddingCleanupDialog(null);
|
|
715
|
+
setEmbeddingCleanupBusy(null);
|
|
716
|
+
setEmbeddingCleanupNote(null);
|
|
717
|
+
}}
|
|
718
|
+
variant="outline"
|
|
719
|
+
>
|
|
720
|
+
Cancel
|
|
721
|
+
</Button>
|
|
722
|
+
<Button
|
|
723
|
+
disabled={embeddingCleanupBusy !== null}
|
|
724
|
+
onClick={() => void handleEmbeddingCleanup("stale")}
|
|
725
|
+
variant="outline"
|
|
726
|
+
>
|
|
727
|
+
{embeddingCleanupBusy === "stale" ? (
|
|
728
|
+
<Loader2Icon className="mr-1.5 size-4 animate-spin" />
|
|
729
|
+
) : null}
|
|
730
|
+
Clear stale
|
|
731
|
+
</Button>
|
|
732
|
+
<Button
|
|
733
|
+
disabled={embeddingCleanupBusy !== null}
|
|
734
|
+
onClick={() => void handleEmbeddingCleanup("all")}
|
|
735
|
+
variant="destructive"
|
|
736
|
+
>
|
|
737
|
+
{embeddingCleanupBusy === "all" ? (
|
|
738
|
+
<Loader2Icon className="mr-1.5 size-4 animate-spin" />
|
|
739
|
+
) : null}
|
|
740
|
+
Clear all
|
|
741
|
+
</Button>
|
|
742
|
+
</DialogFooter>
|
|
743
|
+
</DialogContent>
|
|
744
|
+
</Dialog>
|
|
745
|
+
|
|
608
746
|
{/* Remove confirmation dialog */}
|
|
609
747
|
<Dialog
|
|
610
748
|
onOpenChange={(open) => !open && setRemoveDialog(null)}
|
package/src/serve/routes/api.ts
CHANGED
|
@@ -82,6 +82,7 @@ import {
|
|
|
82
82
|
getModelConfig,
|
|
83
83
|
getPreset,
|
|
84
84
|
listPresets,
|
|
85
|
+
resolveModelUri,
|
|
85
86
|
} from "../../llm/registry";
|
|
86
87
|
import {
|
|
87
88
|
generateGroundedAnswer,
|
|
@@ -212,6 +213,10 @@ export interface UpdateCollectionRequestBody {
|
|
|
212
213
|
};
|
|
213
214
|
}
|
|
214
215
|
|
|
216
|
+
export interface ClearCollectionEmbeddingsRequestBody {
|
|
217
|
+
mode?: "stale" | "all";
|
|
218
|
+
}
|
|
219
|
+
|
|
215
220
|
export interface CollectionResponse {
|
|
216
221
|
name: string;
|
|
217
222
|
path: string;
|
|
@@ -987,6 +992,62 @@ export async function handleUpdateCollection(
|
|
|
987
992
|
});
|
|
988
993
|
}
|
|
989
994
|
|
|
995
|
+
/**
|
|
996
|
+
* POST /api/collections/:name/embeddings/clear
|
|
997
|
+
* Clear stale or all embeddings for a collection.
|
|
998
|
+
*/
|
|
999
|
+
export async function handleClearCollectionEmbeddings(
|
|
1000
|
+
ctxHolder: ContextHolder,
|
|
1001
|
+
store: SqliteAdapter,
|
|
1002
|
+
name: string,
|
|
1003
|
+
req: Request
|
|
1004
|
+
): Promise<Response> {
|
|
1005
|
+
let body: ClearCollectionEmbeddingsRequestBody;
|
|
1006
|
+
try {
|
|
1007
|
+
body = (await req.json()) as ClearCollectionEmbeddingsRequestBody;
|
|
1008
|
+
} catch {
|
|
1009
|
+
return errorResponse("VALIDATION", "Invalid JSON body");
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
const mode = body.mode ?? "stale";
|
|
1013
|
+
if (mode !== "stale" && mode !== "all") {
|
|
1014
|
+
return errorResponse("VALIDATION", "mode must be 'stale' or 'all'");
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
const collection = ctxHolder.config.collections.find(
|
|
1018
|
+
(item) => item.name === name.toLowerCase()
|
|
1019
|
+
);
|
|
1020
|
+
if (!collection) {
|
|
1021
|
+
return errorResponse("NOT_FOUND", `Collection not found: ${name}`, 404);
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
const activeModel = resolveModelUri(
|
|
1025
|
+
ctxHolder.config,
|
|
1026
|
+
"embed",
|
|
1027
|
+
undefined,
|
|
1028
|
+
collection.name
|
|
1029
|
+
);
|
|
1030
|
+
const result = await store.clearEmbeddingsForCollection(collection.name, {
|
|
1031
|
+
mode,
|
|
1032
|
+
activeModel,
|
|
1033
|
+
});
|
|
1034
|
+
if (!result.ok) {
|
|
1035
|
+
const status = result.error.code === "INVALID_INPUT" ? 400 : 500;
|
|
1036
|
+
return errorResponse(result.error.code, result.error.message, status);
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
return jsonResponse({
|
|
1040
|
+
success: true,
|
|
1041
|
+
stats: result.value,
|
|
1042
|
+
note:
|
|
1043
|
+
mode === "all"
|
|
1044
|
+
? `Run gno embed --collection ${collection.name} to rebuild active embeddings.`
|
|
1045
|
+
: result.value.protectedSharedVectors > 0
|
|
1046
|
+
? "Some shared vectors were retained because other active collections still use the same content."
|
|
1047
|
+
: undefined,
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
|
|
990
1051
|
/**
|
|
991
1052
|
* POST /api/sync
|
|
992
1053
|
* Trigger re-index of all or specific collection.
|
|
@@ -3356,6 +3417,7 @@ export async function handleSetPreset(
|
|
|
3356
3417
|
}
|
|
3357
3418
|
|
|
3358
3419
|
console.log(`Switching to preset: ${preset.name}`);
|
|
3420
|
+
const previousEmbedModel = resolveModelUri(ctxHolder.config, "embed");
|
|
3359
3421
|
|
|
3360
3422
|
const syncResult = await (deps?.applyConfigChangeFn ?? applyConfigChange)(
|
|
3361
3423
|
ctxHolder,
|
|
@@ -3405,6 +3467,11 @@ export async function handleSetPreset(
|
|
|
3405
3467
|
return jsonResponse({
|
|
3406
3468
|
success: true,
|
|
3407
3469
|
activePreset: body.presetId,
|
|
3470
|
+
embedModelChanged: previousEmbedModel !== preset.embed,
|
|
3471
|
+
note:
|
|
3472
|
+
previousEmbedModel !== preset.embed
|
|
3473
|
+
? "Embedding model changed. Existing collections may need gno embed so vector results catch up."
|
|
3474
|
+
: undefined,
|
|
3408
3475
|
capabilities: ctxHolder.current.capabilities,
|
|
3409
3476
|
});
|
|
3410
3477
|
}
|
package/src/serve/server.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
handleAsk,
|
|
18
18
|
handleBrowseTree,
|
|
19
19
|
handleCapabilities,
|
|
20
|
+
handleClearCollectionEmbeddings,
|
|
20
21
|
handleCollections,
|
|
21
22
|
handleConnectors,
|
|
22
23
|
handleCreateFolder,
|
|
@@ -547,6 +548,25 @@ export async function startServer(
|
|
|
547
548
|
);
|
|
548
549
|
},
|
|
549
550
|
},
|
|
551
|
+
"/api/collections/:name/embeddings/clear": {
|
|
552
|
+
POST: async (req: Request) => {
|
|
553
|
+
if (!isRequestAllowed(req, port)) {
|
|
554
|
+
return withSecurityHeaders(forbiddenResponse(), isDev);
|
|
555
|
+
}
|
|
556
|
+
const url = new URL(req.url);
|
|
557
|
+
const parts = url.pathname.split("/");
|
|
558
|
+
const name = decodeURIComponent(parts.at(-3) || "");
|
|
559
|
+
return withSecurityHeaders(
|
|
560
|
+
await handleClearCollectionEmbeddings(
|
|
561
|
+
ctxHolder,
|
|
562
|
+
store,
|
|
563
|
+
name,
|
|
564
|
+
req
|
|
565
|
+
),
|
|
566
|
+
isDev
|
|
567
|
+
);
|
|
568
|
+
},
|
|
569
|
+
},
|
|
550
570
|
"/api/doc/:id/links": {
|
|
551
571
|
GET: async (req: Request) => {
|
|
552
572
|
const url = new URL(req.url);
|
package/src/serve/status.ts
CHANGED
|
@@ -15,7 +15,7 @@ import type {
|
|
|
15
15
|
import { getModelsCachePath } from "../app/constants";
|
|
16
16
|
import { ModelCache } from "../llm/cache";
|
|
17
17
|
import { envIsSet, resolveDownloadPolicy } from "../llm/policy";
|
|
18
|
-
import { getActivePreset } from "../llm/registry";
|
|
18
|
+
import { getActivePreset, resolveModelUri } from "../llm/registry";
|
|
19
19
|
import { downloadState, type ServerContext } from "./context";
|
|
20
20
|
|
|
21
21
|
const GIGABYTE = 1024 * 1024 * 1024;
|
|
@@ -615,7 +615,9 @@ export async function buildAppStatus(
|
|
|
615
615
|
ctx: ServerContext,
|
|
616
616
|
deps: StatusBuildDeps = {}
|
|
617
617
|
): Promise<AppStatusResponse> {
|
|
618
|
-
const result = await ctx.store.getStatus(
|
|
618
|
+
const result = await ctx.store.getStatus({
|
|
619
|
+
embedModel: resolveModelUri(ctx.config, "embed"),
|
|
620
|
+
});
|
|
619
621
|
if (!result.ok) {
|
|
620
622
|
throw result.error;
|
|
621
623
|
}
|
|
@@ -24,6 +24,7 @@ import type {
|
|
|
24
24
|
DocLinkSource,
|
|
25
25
|
DocumentInput,
|
|
26
26
|
DocumentRow,
|
|
27
|
+
EmbeddingCleanupStats,
|
|
27
28
|
FtsResult,
|
|
28
29
|
FtsSearchOptions,
|
|
29
30
|
GetGraphOptions,
|
|
@@ -45,6 +46,7 @@ import { buildUri, deriveDocid } from "../../app/constants";
|
|
|
45
46
|
import { normalizeWikiName, stripWikiMdExt } from "../../core/links";
|
|
46
47
|
import { migrations, runMigrations } from "../migrations";
|
|
47
48
|
import { err, ok } from "../types";
|
|
49
|
+
import { modelTableName } from "../vector/sqlite-vec";
|
|
48
50
|
import { loadFts5Snowball } from "./fts5-snowball";
|
|
49
51
|
|
|
50
52
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -2696,9 +2698,12 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2696
2698
|
// Status
|
|
2697
2699
|
// ─────────────────────────────────────────────────────────────────────────
|
|
2698
2700
|
|
|
2699
|
-
async getStatus(
|
|
2701
|
+
async getStatus(options?: {
|
|
2702
|
+
embedModel?: string;
|
|
2703
|
+
}): Promise<StoreResult<IndexStatus>> {
|
|
2700
2704
|
try {
|
|
2701
2705
|
const db = this.ensureOpen();
|
|
2706
|
+
const embedModel = options?.embedModel ?? null;
|
|
2702
2707
|
|
|
2703
2708
|
// Get version
|
|
2704
2709
|
const versionRow = db
|
|
@@ -2729,7 +2734,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2729
2734
|
}
|
|
2730
2735
|
|
|
2731
2736
|
const collectionStats = db
|
|
2732
|
-
.query<CollectionStat, []>(
|
|
2737
|
+
.query<CollectionStat, [string | null, string | null]>(
|
|
2733
2738
|
`
|
|
2734
2739
|
SELECT
|
|
2735
2740
|
c.name,
|
|
@@ -2741,15 +2746,26 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2741
2746
|
(SELECT COUNT(*) FROM content_chunks cc
|
|
2742
2747
|
JOIN documents d2 ON d2.mirror_hash = cc.mirror_hash
|
|
2743
2748
|
WHERE d2.collection = c.name AND d2.active = 1) as chunk_count,
|
|
2744
|
-
(SELECT COUNT(*) FROM
|
|
2745
|
-
|
|
2746
|
-
|
|
2749
|
+
(SELECT COUNT(*) FROM content_chunks cc
|
|
2750
|
+
WHERE EXISTS (
|
|
2751
|
+
SELECT 1 FROM documents d3
|
|
2752
|
+
WHERE d3.collection = c.name
|
|
2753
|
+
AND d3.active = 1
|
|
2754
|
+
AND d3.mirror_hash = cc.mirror_hash
|
|
2755
|
+
)
|
|
2756
|
+
AND EXISTS (
|
|
2757
|
+
SELECT 1 FROM content_vectors cv
|
|
2758
|
+
WHERE cv.mirror_hash = cc.mirror_hash
|
|
2759
|
+
AND cv.seq = cc.seq
|
|
2760
|
+
AND (? IS NULL OR cv.model = ?)
|
|
2761
|
+
AND cv.embedded_at >= cc.created_at
|
|
2762
|
+
)) as embedded_count
|
|
2747
2763
|
FROM collections c
|
|
2748
2764
|
LEFT JOIN documents d ON d.collection = c.name
|
|
2749
2765
|
GROUP BY c.name, c.path
|
|
2750
2766
|
`
|
|
2751
2767
|
)
|
|
2752
|
-
.all();
|
|
2768
|
+
.all(embedModel, embedModel);
|
|
2753
2769
|
|
|
2754
2770
|
// Get totals
|
|
2755
2771
|
const totalsRow = db
|
|
@@ -2773,7 +2789,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2773
2789
|
// Embedding backlog: chunks from active docs without vectors
|
|
2774
2790
|
// Uses EXISTS to avoid duplicates when multiple docs share mirror_hash
|
|
2775
2791
|
const backlogRow = db
|
|
2776
|
-
.query<{ count: number }, []>(
|
|
2792
|
+
.query<{ count: number }, [string | null, string | null]>(
|
|
2777
2793
|
`
|
|
2778
2794
|
SELECT COUNT(*) as count FROM content_chunks c
|
|
2779
2795
|
WHERE EXISTS (
|
|
@@ -2782,11 +2798,14 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2782
2798
|
)
|
|
2783
2799
|
AND NOT EXISTS (
|
|
2784
2800
|
SELECT 1 FROM content_vectors v
|
|
2785
|
-
WHERE v.mirror_hash = c.mirror_hash
|
|
2801
|
+
WHERE v.mirror_hash = c.mirror_hash
|
|
2802
|
+
AND v.seq = c.seq
|
|
2803
|
+
AND (? IS NULL OR v.model = ?)
|
|
2804
|
+
AND v.embedded_at >= c.created_at
|
|
2786
2805
|
)
|
|
2787
2806
|
`
|
|
2788
2807
|
)
|
|
2789
|
-
.get();
|
|
2808
|
+
.get(embedModel, embedModel);
|
|
2790
2809
|
|
|
2791
2810
|
// Recent errors (last 24h)
|
|
2792
2811
|
const recentErrorsRow = db
|
|
@@ -2960,6 +2979,126 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2960
2979
|
);
|
|
2961
2980
|
}
|
|
2962
2981
|
}
|
|
2982
|
+
|
|
2983
|
+
async clearEmbeddingsForCollection(
|
|
2984
|
+
collection: string,
|
|
2985
|
+
options: { mode: "stale" | "all"; activeModel?: string }
|
|
2986
|
+
): Promise<StoreResult<EmbeddingCleanupStats>> {
|
|
2987
|
+
try {
|
|
2988
|
+
const db = this.ensureOpen();
|
|
2989
|
+
const collectionName = collection.toLowerCase();
|
|
2990
|
+
|
|
2991
|
+
if (options.mode === "stale" && !options.activeModel) {
|
|
2992
|
+
return err(
|
|
2993
|
+
"INVALID_INPUT",
|
|
2994
|
+
"activeModel is required for stale embedding cleanup"
|
|
2995
|
+
);
|
|
2996
|
+
}
|
|
2997
|
+
|
|
2998
|
+
const filterSql = options.mode === "stale" ? "AND cv.model != ?" : "";
|
|
2999
|
+
const filterParams =
|
|
3000
|
+
options.mode === "stale" ? [options.activeModel ?? ""] : [];
|
|
3001
|
+
|
|
3002
|
+
const deletableRows = db
|
|
3003
|
+
.query<{ mirror_hash: string; model: string; seq: number }, string[]>(
|
|
3004
|
+
`
|
|
3005
|
+
SELECT DISTINCT cv.mirror_hash, cv.seq, cv.model
|
|
3006
|
+
FROM content_vectors cv
|
|
3007
|
+
WHERE EXISTS (
|
|
3008
|
+
SELECT 1 FROM documents d
|
|
3009
|
+
WHERE d.collection = ?
|
|
3010
|
+
AND d.mirror_hash = cv.mirror_hash
|
|
3011
|
+
)
|
|
3012
|
+
${filterSql}
|
|
3013
|
+
AND NOT EXISTS (
|
|
3014
|
+
SELECT 1 FROM documents d2
|
|
3015
|
+
WHERE d2.mirror_hash = cv.mirror_hash
|
|
3016
|
+
AND d2.collection != ?
|
|
3017
|
+
AND d2.active = 1
|
|
3018
|
+
)
|
|
3019
|
+
`
|
|
3020
|
+
)
|
|
3021
|
+
.all(collectionName, ...filterParams, collectionName);
|
|
3022
|
+
|
|
3023
|
+
const protectedRow = db
|
|
3024
|
+
.query<{ count: number }, string[]>(
|
|
3025
|
+
`
|
|
3026
|
+
SELECT COUNT(*) as count
|
|
3027
|
+
FROM (
|
|
3028
|
+
SELECT DISTINCT cv.mirror_hash, cv.seq, cv.model
|
|
3029
|
+
FROM content_vectors cv
|
|
3030
|
+
WHERE EXISTS (
|
|
3031
|
+
SELECT 1 FROM documents d
|
|
3032
|
+
WHERE d.collection = ?
|
|
3033
|
+
AND d.mirror_hash = cv.mirror_hash
|
|
3034
|
+
)
|
|
3035
|
+
${filterSql}
|
|
3036
|
+
AND EXISTS (
|
|
3037
|
+
SELECT 1 FROM documents d2
|
|
3038
|
+
WHERE d2.mirror_hash = cv.mirror_hash
|
|
3039
|
+
AND d2.collection != ?
|
|
3040
|
+
AND d2.active = 1
|
|
3041
|
+
)
|
|
3042
|
+
)
|
|
3043
|
+
`
|
|
3044
|
+
)
|
|
3045
|
+
.get(collectionName, ...filterParams, collectionName);
|
|
3046
|
+
|
|
3047
|
+
const deletedModels = [...new Set(deletableRows.map((row) => row.model))];
|
|
3048
|
+
|
|
3049
|
+
const transaction = db.transaction(() => {
|
|
3050
|
+
const deleteVectorStmt = db.prepare(
|
|
3051
|
+
`DELETE FROM content_vectors WHERE mirror_hash = ? AND seq = ? AND model = ?`
|
|
3052
|
+
);
|
|
3053
|
+
const vecDeleteStatements = new Map<
|
|
3054
|
+
string,
|
|
3055
|
+
ReturnType<typeof db.prepare> | null
|
|
3056
|
+
>();
|
|
3057
|
+
|
|
3058
|
+
for (const row of deletableRows) {
|
|
3059
|
+
deleteVectorStmt.run(row.mirror_hash, row.seq, row.model);
|
|
3060
|
+
|
|
3061
|
+
let deleteVecStmt = vecDeleteStatements.get(row.model);
|
|
3062
|
+
if (deleteVecStmt === undefined) {
|
|
3063
|
+
try {
|
|
3064
|
+
deleteVecStmt = db.prepare(
|
|
3065
|
+
`DELETE FROM ${modelTableName(row.model)} WHERE chunk_id = ?`
|
|
3066
|
+
);
|
|
3067
|
+
} catch {
|
|
3068
|
+
deleteVecStmt = null;
|
|
3069
|
+
}
|
|
3070
|
+
vecDeleteStatements.set(row.model, deleteVecStmt);
|
|
3071
|
+
}
|
|
3072
|
+
|
|
3073
|
+
if (deleteVecStmt) {
|
|
3074
|
+
try {
|
|
3075
|
+
deleteVecStmt.run(`${row.mirror_hash}:${row.seq}`);
|
|
3076
|
+
} catch {
|
|
3077
|
+
// Best effort; a later vec sync/rebuild can recover.
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
}
|
|
3081
|
+
});
|
|
3082
|
+
|
|
3083
|
+
transaction();
|
|
3084
|
+
|
|
3085
|
+
return ok({
|
|
3086
|
+
collection: collectionName,
|
|
3087
|
+
deletedVectors: deletableRows.length,
|
|
3088
|
+
deletedModels,
|
|
3089
|
+
mode: options.mode,
|
|
3090
|
+
protectedSharedVectors: protectedRow?.count ?? 0,
|
|
3091
|
+
});
|
|
3092
|
+
} catch (cause) {
|
|
3093
|
+
return err(
|
|
3094
|
+
"QUERY_FAILED",
|
|
3095
|
+
cause instanceof Error
|
|
3096
|
+
? cause.message
|
|
3097
|
+
: "Failed to clear collection embeddings",
|
|
3098
|
+
cause
|
|
3099
|
+
);
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
2963
3102
|
}
|
|
2964
3103
|
|
|
2965
3104
|
// ─────────────────────────────────────────────────────────────────────────────
|
package/src/store/types.ts
CHANGED
|
@@ -403,6 +403,14 @@ export interface CleanupStats {
|
|
|
403
403
|
expiredCache: number;
|
|
404
404
|
}
|
|
405
405
|
|
|
406
|
+
export interface EmbeddingCleanupStats {
|
|
407
|
+
collection: string;
|
|
408
|
+
deletedVectors: number;
|
|
409
|
+
deletedModels: string[];
|
|
410
|
+
mode: "stale" | "all";
|
|
411
|
+
protectedSharedVectors: number;
|
|
412
|
+
}
|
|
413
|
+
|
|
406
414
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
407
415
|
// Graph Types
|
|
408
416
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -846,7 +854,9 @@ export interface StorePort {
|
|
|
846
854
|
/**
|
|
847
855
|
* Get index status with counts and health info.
|
|
848
856
|
*/
|
|
849
|
-
getStatus(
|
|
857
|
+
getStatus(options?: {
|
|
858
|
+
embedModel?: string;
|
|
859
|
+
}): Promise<StoreResult<IndexStatus>>;
|
|
850
860
|
|
|
851
861
|
// ─────────────────────────────────────────────────────────────────────────
|
|
852
862
|
// Errors
|
|
@@ -870,4 +880,12 @@ export interface StorePort {
|
|
|
870
880
|
* Remove orphaned content, chunks, vectors, and expired cache.
|
|
871
881
|
*/
|
|
872
882
|
cleanupOrphans(): Promise<StoreResult<CleanupStats>>;
|
|
883
|
+
|
|
884
|
+
/**
|
|
885
|
+
* Remove embeddings for a collection.
|
|
886
|
+
*/
|
|
887
|
+
clearEmbeddingsForCollection(
|
|
888
|
+
collection: string,
|
|
889
|
+
options: { mode: "stale" | "all"; activeModel?: string }
|
|
890
|
+
): Promise<StoreResult<EmbeddingCleanupStats>>;
|
|
873
891
|
}
|
|
@@ -51,7 +51,7 @@ export function decodeEmbedding(blob: Uint8Array): Float32Array {
|
|
|
51
51
|
* Generate deterministic table name from model URI.
|
|
52
52
|
* First 8 chars of SHA256 hash.
|
|
53
53
|
*/
|
|
54
|
-
function modelTableName(modelUri: string): string {
|
|
54
|
+
export function modelTableName(modelUri: string): string {
|
|
55
55
|
const hash = createHash("sha256").update(modelUri).digest("hex").slice(0, 8);
|
|
56
56
|
return `vec_${hash}`;
|
|
57
57
|
}
|