@gmickel/gno 2.8.2 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.2.zip → gno-browser-clipper-v2.8.3.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +9 -1
- package/spec/cli.md +66 -4
- package/spec/mcp.md +5 -6
- package/src/cli/commands/daemon.ts +7 -2
- package/src/cli/commands/doctor.ts +42 -24
- package/src/cli/commands/index-cmd.ts +10 -2
- package/src/cli/commands/links.ts +5 -22
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/status.ts +60 -0
- package/src/cli/commands/update.ts +6 -1
- package/src/cli/commands/vec.ts +42 -2
- package/src/cli/program.ts +2 -2
- package/src/config/types.ts +19 -0
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
- package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
- package/src/converters/adapters/xlsx/adapter.ts +132 -0
- package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
- package/src/converters/registry.ts +62 -6
- package/src/converters/versions.ts +2 -0
- package/src/core/audit-freshness.ts +1 -1
- package/src/core/audit-outside-index.ts +48 -3
- package/src/core/change-diff.ts +16 -4
- package/src/core/link-destination-parse.ts +21 -4
- package/src/core/link-inventory-markdown.ts +23 -3
- package/src/core/links.ts +74 -16
- package/src/core/network-boundary-inventory.ts +18 -0
- package/src/index.ts +10 -0
- package/src/ingestion/document-metadata.ts +256 -0
- package/src/ingestion/file-child-env.ts +9 -0
- package/src/ingestion/file-child.ts +106 -0
- package/src/ingestion/file-processor.ts +546 -0
- package/src/ingestion/frontmatter.ts +6 -2
- package/src/ingestion/index.ts +1 -0
- package/src/ingestion/prepare-convert.ts +32 -0
- package/src/ingestion/prepare-file.ts +272 -0
- package/src/ingestion/prepare-worker.ts +43 -0
- package/src/ingestion/record-container.ts +7 -2
- package/src/ingestion/strip.ts +71 -8
- package/src/ingestion/sync-options.ts +7 -1
- package/src/ingestion/sync.ts +207 -400
- package/src/ingestion/types.ts +18 -0
- package/src/llm/native-worker/embedding-identity.ts +15 -3
- package/src/mcp/http-transport.ts +11 -0
- package/src/mcp/tools/links.ts +10 -38
- package/src/serve/resident-request.ts +19 -0
- package/src/serve/resident-runtime.ts +84 -8
- package/src/serve/routes/links.ts +6 -24
- package/src/serve/server.ts +3 -2
- package/src/sessions/config-refresh.ts +3 -2
- package/src/store/sqlite/graph-similarity.ts +5 -13
- package/src/store/vector/partition-index.ts +100 -0
- package/src/store/vector/stored-vectors.ts +20 -15
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.2.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
|
|
|
139
139
|
|
|
140
140
|
<!-- public-truth:current-version -->
|
|
141
141
|
|
|
142
|
-
> Current source version: **v2.8.
|
|
142
|
+
> Current source version: **v2.8.3**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
143
143
|
|
|
144
144
|
<!-- /public-truth -->
|
|
145
145
|
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
7fe15ce08a24f4dae4ac6694f62bb26d94bbc47d6abc09826f29267ace10558d gno-browser-clipper-v2.8.3.zip
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gmickel/gno",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.3",
|
|
4
4
|
"description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"embeddings",
|
|
@@ -197,7 +197,15 @@
|
|
|
197
197
|
"mammoth": "1.12.2",
|
|
198
198
|
"mdast-util-from-markdown": "2.0.3",
|
|
199
199
|
"mdast-util-gfm": "3.1.0",
|
|
200
|
+
"mdast-util-gfm-autolink-literal": "2.0.1",
|
|
201
|
+
"mdast-util-gfm-footnote": "2.1.0",
|
|
202
|
+
"mdast-util-gfm-strikethrough": "2.0.0",
|
|
203
|
+
"mdast-util-gfm-task-list-item": "2.0.0",
|
|
200
204
|
"micromark-extension-gfm": "3.0.0",
|
|
205
|
+
"micromark-extension-gfm-autolink-literal": "2.1.0",
|
|
206
|
+
"micromark-extension-gfm-footnote": "2.1.0",
|
|
207
|
+
"micromark-extension-gfm-strikethrough": "2.1.0",
|
|
208
|
+
"micromark-extension-gfm-task-list-item": "2.1.0",
|
|
201
209
|
"mime-types": "2.1.35",
|
|
202
210
|
"minimatch": "10.2.6",
|
|
203
211
|
"nanoid": "6.0.1",
|
package/spec/cli.md
CHANGED
|
@@ -987,7 +987,10 @@ resolves to a regular file inside the workspace, at most 200,000 files), no
|
|
|
987
987
|
file is opened, and no graph edge is created. Wiki targets match files by the workspace
|
|
988
988
|
link rules; a non-Markdown target needs its extension, `.md` is optional. A
|
|
989
989
|
Markdown link matches only the file at its resolved path (NFC,
|
|
990
|
-
case-insensitive)
|
|
990
|
+
case-insensitive); as for wiki links, `.md` is optional, so `[x](Note)` matches
|
|
991
|
+
an unindexed `Note.md`. An indexed `Note.md` (compared under the same NFC,
|
|
992
|
+
Unicode-lowercase key) is not outside the index, and Markdown link resolution
|
|
993
|
+
itself stays exact-path, so that link remains in `links.local-targets`. A
|
|
991
994
|
target missing from the listing stays unresolved; an unreadable folder or the
|
|
992
995
|
file bound makes the listing incomplete, which the `links.outside-index` rule
|
|
993
996
|
message states once.
|
|
@@ -995,7 +998,12 @@ message states once.
|
|
|
995
998
|
The wiki link parser reads Obsidian's table-escaped alias `[[Note\|Alias]]`
|
|
996
999
|
as target `Note` with alias `Alias`. Markdown link text may contain balanced
|
|
997
1000
|
square brackets (`[see [1]](note.md)`); a destination containing a square
|
|
998
|
-
bracket is not a link.
|
|
1001
|
+
bracket is not a link. Markdown links follow CommonMark, as Obsidian renders
|
|
1002
|
+
them: a `[` escaped with a backslash (`\[clause](see section 4)`) opens no
|
|
1003
|
+
link, and an inline destination with an unescaped space is plain text unless
|
|
1004
|
+
it is wrapped in angle brackets. `[x](<my note.md>)` and `[x](my%20note.md)`
|
|
1005
|
+
both link to `my note.md`; a link title (`[x](note.md "Title")`) is not part
|
|
1006
|
+
of the target.
|
|
999
1007
|
|
|
1000
1008
|
The JSON contract is versioned as `gno://schemas/audit-report@1.0`. Finding IDs
|
|
1001
1009
|
are stable SHA-256 identities derived from rule, normalized subject/location,
|
|
@@ -1472,6 +1480,33 @@ individual virtual document. The closed `record.adapter` identity contains the
|
|
|
1472
1480
|
adapter ID, version, and configuration fingerprint. `get` and `multi-get`
|
|
1473
1481
|
preserve the same fields.
|
|
1474
1482
|
|
|
1483
|
+
Every file is indexed under the per-file budget `conversion.timeoutMs`
|
|
1484
|
+
(default 60000) and `conversion.maxMemoryMb` (default half of physical memory,
|
|
1485
|
+
at least 2048; process resident memory). Conversion and every post-conversion
|
|
1486
|
+
phase (metadata, code regions, change-journal structure, chunking, link
|
|
1487
|
+
extraction) run in one worker per process, one file at a time; the worker is
|
|
1488
|
+
terminated at either limit, mid-phase, and replaced for the next file.
|
|
1489
|
+
Database reads and writes stay on the main thread. A standalone compiled
|
|
1490
|
+
executable, which cannot start the worker, runs the phases in a child process
|
|
1491
|
+
of itself (internal environment flag) under the same limits; the child's
|
|
1492
|
+
resident memory counts toward `conversion.maxMemoryMb` (Linux `/proc`, `ps`
|
|
1493
|
+
elsewhere; time only on Windows). A processor that cannot start fails the file
|
|
1494
|
+
with `ISOLATION_UNAVAILABLE`; nothing is prepared without the budget. The
|
|
1495
|
+
child is SIGKILLed on process exit, SIGTERM, SIGINT (via the CLI's exit
|
|
1496
|
+
path) and resident shutdown; on Linux it also carries a parent-death signal,
|
|
1497
|
+
so it dies with a SIGKILLed parent, and elsewhere it exits at its next step
|
|
1498
|
+
once its parent is gone.
|
|
1499
|
+
A file over either limit is recorded with error code `TIMEOUT` or
|
|
1500
|
+
`MEMORY_LIMIT` (the message names the running phase), stays unindexed, and is
|
|
1501
|
+
retried by the next sync; the rest of the run continues. `update` and `index` always print such a
|
|
1502
|
+
file in terminal output as `[CODE] <relPath>: <message>` (other per-file
|
|
1503
|
+
errors stay `--verbose` only), and write
|
|
1504
|
+
`Still converting <collection>/<relPath> after <n>s (budget <m>s)` to stderr
|
|
1505
|
+
once a file has been processing for 10 seconds or half its budget, whichever
|
|
1506
|
+
is sooner.
|
|
1507
|
+
`gno status` lists stopped files under "Stopped at conversion budget" in
|
|
1508
|
+
terminal output (JSON output is unchanged).
|
|
1509
|
+
|
|
1475
1510
|
File/export adapter iteration has a 60-second deadline in addition to the byte,
|
|
1476
1511
|
record, metadata, and total-character caps. A partial import that retains valid
|
|
1477
1512
|
siblings exits successfully but is never silent: terminal output includes the
|
|
@@ -2976,7 +3011,11 @@ misconfigured `findings` block (see [Daemon Mode](../docs/DAEMON.md#scheduled-fi
|
|
|
2976
3011
|
The `embedding-fingerprint` check is additive doctor-only diagnostics. It uses
|
|
2977
3012
|
the active embed model and stored vector dimensions to report the current
|
|
2978
3013
|
freshness fingerprint, pending/stale chunks, legacy empty-fingerprint vectors,
|
|
2979
|
-
and stored fingerprint groups.
|
|
3014
|
+
and stored fingerprint groups. On an index with an activated vector partition
|
|
3015
|
+
for the embed model (every index `gno embed` wrote since 2.7), the groups are
|
|
3016
|
+
that partition with its current chunk count and there are no legacy vectors;
|
|
3017
|
+
legacy `content_vectors` groups are read only before any partition activates.
|
|
3018
|
+
Stale, legacy, and mixed groups are warnings;
|
|
2980
3019
|
recover with `gno embed`, or `gno embed --force` if vectors still look stale.
|
|
2981
3020
|
|
|
2982
3021
|
The additive `activation` object uses the same contract as `gno status` and
|
|
@@ -3988,7 +4027,10 @@ gno similar <doc> [-n, --limit <num>] [--threshold <num>] [--cross-collection] [
|
|
|
3988
4027
|
|
|
3989
4028
|
- Finds documents semantically similar to the source document
|
|
3990
4029
|
- Requires embeddings to be generated (`gno embed`)
|
|
3991
|
-
-
|
|
4030
|
+
- Source vector: the stored vector of the document's first chunk (the
|
|
4031
|
+
lowest-seq chunk whose vector is current), read from the active vector
|
|
4032
|
+
partition. `gno_similar` (MCP), `GET /api/doc/:id/similar` (REST) and graph
|
|
4033
|
+
similarity edges use the same rule, so scores match across surfaces
|
|
3992
4034
|
- By default, limits results to same collection
|
|
3993
4035
|
|
|
3994
4036
|
**Output (JSON):**
|
|
@@ -4203,6 +4245,9 @@ gno changes --follow --jsonl [--cursor <cursor>] [--collection <name>]
|
|
|
4203
4245
|
old/new identity and hash snapshots, normalized structural deltas, pagination,
|
|
4204
4246
|
cursor-expiry, and retention-truncation disclosure.
|
|
4205
4247
|
- The journal never returns source bodies.
|
|
4248
|
+
- Link additions and removals in `structureDelta.links` are recorded only for
|
|
4249
|
+
Markdown sources, matching link extraction; content changes of other
|
|
4250
|
+
documents (code, text, converted files, records) are still journaled.
|
|
4206
4251
|
|
|
4207
4252
|
**Follow mode (`--follow --jsonl`)** streams journal events as they land and is
|
|
4208
4253
|
the durable automation input for consumers that resume across restarts.
|
|
@@ -4444,6 +4489,21 @@ gno serve --detach --stop
|
|
|
4444
4489
|
|
|
4445
4490
|
---
|
|
4446
4491
|
|
|
4492
|
+
#### Config reload while running
|
|
4493
|
+
|
|
4494
|
+
A running `gno serve` or `gno daemon` re-reads its config file before each
|
|
4495
|
+
request (REST, Web UI, HTTP MCP, daemon status) and every 2 seconds, and adopts
|
|
4496
|
+
changes to `collections` and `contexts` made by the CLI (`gno collection
|
|
4497
|
+
add/remove`) or by hand: status, MCP `gno_status`, the watcher and search scope
|
|
4498
|
+
follow without a restart. A newly added collection is watched from then on; its
|
|
4499
|
+
existing files are indexed by `gno update` (or the next start's initial sync).
|
|
4500
|
+
An adoption that changes the collection set or policies moves the egress policy
|
|
4501
|
+
epoch as a REST collection change does. The checks are the sessions config
|
|
4502
|
+
refresh's: an unreadable config file answers an error (`500`, sessions code
|
|
4503
|
+
`SESSIONS_RUNTIME_FAILURE`), never the stale config, and a config bound to
|
|
4504
|
+
another index is refused. Other settings (models, gateway, sessions sources
|
|
4505
|
+
outside the sessions routes) keep their restart behaviour.
|
|
4506
|
+
|
|
4447
4507
|
### gno daemon
|
|
4448
4508
|
|
|
4449
4509
|
Start a headless long-running watcher process for continuous indexing.
|
|
@@ -4483,6 +4543,8 @@ is blocked.
|
|
|
4483
4543
|
|
|
4484
4544
|
- Opens DB once at startup
|
|
4485
4545
|
- Loads config and requires at least one configured collection
|
|
4546
|
+
- Follows collection edits to the config file while running (see
|
|
4547
|
+
[Config reload](#config-reload-while-running))
|
|
4486
4548
|
- Starts the same watcher + embed scheduler used by `gno serve`
|
|
4487
4549
|
- Exact contained eligible paths always use targeted content-hash sync.
|
|
4488
4550
|
Ambiguous temp/directory/missing-name/recursive-delete events use bounded,
|
package/spec/mcp.md
CHANGED
|
@@ -2188,12 +2188,11 @@ Find semantically similar documents using vector embeddings.
|
|
|
2188
2188
|
|
|
2189
2189
|
**Algorithm:**
|
|
2190
2190
|
|
|
2191
|
-
1.
|
|
2192
|
-
2.
|
|
2193
|
-
3.
|
|
2194
|
-
4.
|
|
2195
|
-
5.
|
|
2196
|
-
6. Return top N similar documents with scores
|
|
2191
|
+
1. Take the stored vector of the source document's first chunk (the lowest-seq chunk whose vector is current) from the active vector partition (legacy `content_vectors` only before any partition activates); no model is loaded. This is the same source vector `gno similar`, `GET /api/doc/:id/similar` and graph similarity edges use, so scores match across surfaces
|
|
2192
|
+
2. Normalize it to unit length
|
|
2193
|
+
3. Search for nearest neighbors using sqlite-vec
|
|
2194
|
+
4. Exclude self and filter by collection if not crossCollection
|
|
2195
|
+
5. Return top N similar documents with scores
|
|
2197
2196
|
|
|
2198
2197
|
**Errors:**
|
|
2199
2198
|
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
resolveHttpGatewayConfig,
|
|
17
17
|
} from "../../mcp/http-security";
|
|
18
18
|
import { startBackgroundRuntime } from "../../serve/background-runtime";
|
|
19
|
+
import { refreshResidentConfig } from "../../serve/resident-request";
|
|
19
20
|
import { handleResidentStatus, handleStatus } from "../../serve/routes/api";
|
|
20
21
|
import { createMcpHttpGateway } from "../../serve/routes/mcp";
|
|
21
22
|
import { SessionsError } from "../../sessions/types";
|
|
@@ -106,6 +107,8 @@ export const authorizeDaemonStatus = async (
|
|
|
106
107
|
): Promise<Response> => {
|
|
107
108
|
const authorization = await gateway.security.authorize(request, server);
|
|
108
109
|
if (!authorization.ok) return authorization.response;
|
|
110
|
+
const refreshFailure = await refreshResidentConfig(runtime);
|
|
111
|
+
if (refreshFailure) return refreshFailure;
|
|
109
112
|
try {
|
|
110
113
|
enforceCollectionEgress({
|
|
111
114
|
collections: runtime.config.collections,
|
|
@@ -132,13 +135,15 @@ export const authorizeDaemonStatus = async (
|
|
|
132
135
|
}
|
|
133
136
|
};
|
|
134
137
|
|
|
135
|
-
export function handleDaemonAppStatus(
|
|
138
|
+
export async function handleDaemonAppStatus(
|
|
136
139
|
runtime: ResidentRuntime,
|
|
137
140
|
host: string
|
|
138
|
-
): Promise<Response>
|
|
141
|
+
): Promise<Response> {
|
|
139
142
|
if (!isHttpGatewayLoopbackBind(host)) {
|
|
140
143
|
return new Response(null, { status: 404 });
|
|
141
144
|
}
|
|
145
|
+
const refreshFailure = await refreshResidentConfig(runtime);
|
|
146
|
+
if (refreshFailure) return refreshFailure;
|
|
142
147
|
return handleStatus(runtime.ctxHolder.current, {
|
|
143
148
|
getResidentStatus: () => runtime.getStatus(),
|
|
144
149
|
});
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
resolveFindingsSchedule,
|
|
32
32
|
} from "../../core/findings-run-state";
|
|
33
33
|
import { formatVectorPartitionLines } from "../../core/vector-partition-status";
|
|
34
|
+
import { getEmbeddingFingerprint } from "../../embed/fingerprint";
|
|
34
35
|
import { getCodeChunkingStatus } from "../../ingestion/chunker";
|
|
35
36
|
import { ModelCache } from "../../llm/cache";
|
|
36
37
|
import { getActivePreset, resolveModelUri } from "../../llm/registry";
|
|
@@ -42,6 +43,8 @@ import {
|
|
|
42
43
|
getLoadAttempts,
|
|
43
44
|
} from "../../store/sqlite/setup";
|
|
44
45
|
import { getStoredEmbeddingFingerprint } from "../../store/vector/freshness";
|
|
46
|
+
import { currentOwnerCount } from "../../store/vector/runtime-compat";
|
|
47
|
+
import { storedVectorPartition } from "../../store/vector/status";
|
|
45
48
|
import {
|
|
46
49
|
buildDoctorActivation,
|
|
47
50
|
checkConnectorActivation,
|
|
@@ -319,10 +322,14 @@ async function checkEmbeddingFingerprints(
|
|
|
319
322
|
];
|
|
320
323
|
}
|
|
321
324
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
325
|
+
// An index embedded since 2.7 keeps its vectors in the active partition;
|
|
326
|
+
// legacy `content_vectors` count only before any partition activates.
|
|
327
|
+
const partition = storedVectorPartition(db, model);
|
|
328
|
+
const legacyChunks = partition
|
|
329
|
+
? 0
|
|
330
|
+
: (db
|
|
331
|
+
.query<{ count: number }, [string]>(
|
|
332
|
+
`
|
|
326
333
|
SELECT COUNT(*) as count
|
|
327
334
|
FROM content_vectors v
|
|
328
335
|
JOIN content_chunks c
|
|
@@ -336,12 +343,23 @@ async function checkEmbeddingFingerprints(
|
|
|
336
343
|
AND d.active = 1
|
|
337
344
|
)
|
|
338
345
|
`
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const groups =
|
|
343
|
-
|
|
344
|
-
|
|
346
|
+
)
|
|
347
|
+
.get(model)?.count ?? 0);
|
|
348
|
+
|
|
349
|
+
const groups = partition
|
|
350
|
+
? [
|
|
351
|
+
{
|
|
352
|
+
model,
|
|
353
|
+
fingerprint: getEmbeddingFingerprint({
|
|
354
|
+
modelUri: model,
|
|
355
|
+
dimensions: partition.dimensions,
|
|
356
|
+
}),
|
|
357
|
+
count: currentOwnerCount(db, partition.partitionId),
|
|
358
|
+
},
|
|
359
|
+
]
|
|
360
|
+
: db
|
|
361
|
+
.query<{ model: string; fingerprint: string; count: number }, []>(
|
|
362
|
+
`
|
|
345
363
|
SELECT
|
|
346
364
|
v.model as model,
|
|
347
365
|
v.embed_fingerprint as fingerprint,
|
|
@@ -358,18 +376,18 @@ async function checkEmbeddingFingerprints(
|
|
|
358
376
|
GROUP BY v.model, v.embed_fingerprint
|
|
359
377
|
ORDER BY count DESC, v.model ASC, v.embed_fingerprint ASC
|
|
360
378
|
`
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
const mixedGroups =
|
|
379
|
+
)
|
|
380
|
+
.all();
|
|
381
|
+
const fingerprintGroups = groups.map((group) => ({
|
|
382
|
+
model: group.model,
|
|
383
|
+
fingerprint: group.fingerprint,
|
|
384
|
+
count: group.count,
|
|
385
|
+
current:
|
|
386
|
+
group.model === model && group.fingerprint === currentFingerprint,
|
|
387
|
+
legacy: group.fingerprint === "",
|
|
388
|
+
}));
|
|
389
|
+
|
|
390
|
+
const mixedGroups = fingerprintGroups.length;
|
|
373
391
|
const pendingChunks = statusResult.value.embeddingBacklog;
|
|
374
392
|
const hasWarnings =
|
|
375
393
|
pendingChunks > 0 || legacyChunks > 0 || mixedGroups > 1;
|
|
@@ -380,7 +398,7 @@ async function checkEmbeddingFingerprints(
|
|
|
380
398
|
pendingChunks,
|
|
381
399
|
legacyChunks,
|
|
382
400
|
mixedGroups,
|
|
383
|
-
groups,
|
|
401
|
+
groups: fingerprintGroups,
|
|
384
402
|
};
|
|
385
403
|
|
|
386
404
|
const message =
|
|
@@ -392,7 +410,7 @@ async function checkEmbeddingFingerprints(
|
|
|
392
410
|
if (hasWarnings) {
|
|
393
411
|
details.push("Run: gno embed");
|
|
394
412
|
details.push("If vectors still look stale, run: gno embed --force");
|
|
395
|
-
for (const group of
|
|
413
|
+
for (const group of fingerprintGroups) {
|
|
396
414
|
const label = describeFingerprintGroup(group);
|
|
397
415
|
const fingerprint = group.legacy
|
|
398
416
|
? "(empty)"
|
|
@@ -28,7 +28,11 @@ import {
|
|
|
28
28
|
withContentTypeRules,
|
|
29
29
|
} from "../../ingestion";
|
|
30
30
|
import { type EmbedResult, embedStageOutcome } from "./embed";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
formatSyncResultLines,
|
|
33
|
+
initStore,
|
|
34
|
+
reportSlowConversion,
|
|
35
|
+
} from "./shared";
|
|
32
36
|
|
|
33
37
|
/**
|
|
34
38
|
* Options for index command.
|
|
@@ -202,7 +206,11 @@ export async function index(options: IndexOptions = {}): Promise<IndexResult> {
|
|
|
202
206
|
collections,
|
|
203
207
|
store,
|
|
204
208
|
withContentTypeRules(
|
|
205
|
-
{
|
|
209
|
+
{
|
|
210
|
+
gitPull: options.gitPull,
|
|
211
|
+
runUpdateCmd: true,
|
|
212
|
+
onSlowConversion: reportSlowConversion,
|
|
213
|
+
},
|
|
206
214
|
config
|
|
207
215
|
)
|
|
208
216
|
);
|
|
@@ -587,21 +587,17 @@ export async function similar(
|
|
|
587
587
|
}
|
|
588
588
|
const db = store.getRawDb();
|
|
589
589
|
|
|
590
|
-
//
|
|
590
|
+
// Similarity source vector (first chunk, unit length), active partition
|
|
591
591
|
const {
|
|
592
|
-
|
|
592
|
+
readSimilaritySourceVectors,
|
|
593
593
|
resolveStoredVectorSource,
|
|
594
594
|
similarityHitDocuments,
|
|
595
595
|
storedVectorSearchOptions,
|
|
596
596
|
} = await import("../../store/vector/stored-vectors.js");
|
|
597
597
|
const source = resolveStoredVectorSource(db, modelPreset.embed);
|
|
598
|
-
const
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
source,
|
|
602
|
-
[{ id: doc.id, mirrorHash: doc.mirrorHash }],
|
|
603
|
-
{ firstChunkOnly: true }
|
|
604
|
-
).get(doc.id) ?? [];
|
|
598
|
+
const embedding = readSimilaritySourceVectors(db, source, [
|
|
599
|
+
{ id: doc.id, mirrorHash: doc.mirrorHash },
|
|
600
|
+
]).get(doc.id);
|
|
605
601
|
|
|
606
602
|
if (!embedding) {
|
|
607
603
|
return {
|
|
@@ -610,20 +606,7 @@ export async function similar(
|
|
|
610
606
|
isValidation: true,
|
|
611
607
|
};
|
|
612
608
|
}
|
|
613
|
-
|
|
614
|
-
// Normalize embedding for cosine similarity
|
|
615
609
|
const dimensions = embedding.length;
|
|
616
|
-
let norm = 0;
|
|
617
|
-
for (let i = 0; i < dimensions; i++) {
|
|
618
|
-
const val = embedding[i] ?? 0;
|
|
619
|
-
norm += val * val;
|
|
620
|
-
}
|
|
621
|
-
norm = Math.sqrt(norm);
|
|
622
|
-
if (norm > 0) {
|
|
623
|
-
for (let i = 0; i < dimensions; i++) {
|
|
624
|
-
embedding[i] = (embedding[i] ?? 0) / norm;
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
610
|
|
|
628
611
|
// Create vector index port for search
|
|
629
612
|
const { createVectorIndexPort } =
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { Collection, Config } from "../../config/types";
|
|
9
|
-
import type { SyncResult } from "../../ingestion";
|
|
9
|
+
import type { SlowConversionEvent, SyncResult } from "../../ingestion";
|
|
10
10
|
import type { SearchResults } from "../../pipeline/types";
|
|
11
11
|
|
|
12
12
|
import {
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
loadConfig,
|
|
21
21
|
writeConfigWarningsToStderr,
|
|
22
22
|
} from "../../config";
|
|
23
|
+
import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
|
|
23
24
|
import { SqliteAdapter } from "../../store/sqlite/adapter";
|
|
24
25
|
import { assertCliSessionBinding } from "../session-binding";
|
|
25
26
|
|
|
@@ -164,6 +165,21 @@ export function decorateSearchResultsForIndex(
|
|
|
164
165
|
};
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Name a file whose conversion is still running past the notice threshold,
|
|
170
|
+
* so an operator can see which file a slow run is working on (stderr, so
|
|
171
|
+
* JSON output on stdout stays parseable).
|
|
172
|
+
*/
|
|
173
|
+
export function reportSlowConversion(event: SlowConversionEvent): void {
|
|
174
|
+
process.stderr.write(
|
|
175
|
+
`Still converting ${event.collection}/${event.relPath} after ${formatSeconds(event.elapsedMs)} (budget ${formatSeconds(event.budgetMs)})\n`
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Seconds with one decimal below ten seconds, whole seconds above. */
|
|
180
|
+
const formatSeconds = (ms: number): string =>
|
|
181
|
+
ms < 10_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms / 1000)}s`;
|
|
182
|
+
|
|
167
183
|
/**
|
|
168
184
|
* Format sync result lines (shared between update and index commands).
|
|
169
185
|
*/
|
|
@@ -194,6 +210,15 @@ export function formatSyncResultLines(
|
|
|
194
210
|
if (c.filesErrored > 0) {
|
|
195
211
|
lines.push(` ${c.filesErrored} errors`);
|
|
196
212
|
}
|
|
213
|
+
// A file stopped at its conversion budget is always named, not only in
|
|
214
|
+
// verbose output: it stays unindexed until a later run finishes it.
|
|
215
|
+
if (!options.verbose) {
|
|
216
|
+
for (const err of c.errors) {
|
|
217
|
+
if (BUDGET_ERROR_CODES.has(err.code)) {
|
|
218
|
+
lines.push(` [${err.code}] ${err.relPath}: ${err.message}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
197
222
|
if (c.filesMarkedInactive > 0) {
|
|
198
223
|
lines.push(` ${c.filesMarkedInactive} marked inactive`);
|
|
199
224
|
}
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
formatMemoryStatusLines,
|
|
28
28
|
} from "../../core/memory-diagnostics";
|
|
29
29
|
import { formatVectorPartitionLines } from "../../core/vector-partition-status";
|
|
30
|
+
import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
|
|
30
31
|
import { ModelCache } from "../../llm/cache";
|
|
31
32
|
import { getActivePreset, resolveModelUri } from "../../llm/registry";
|
|
32
33
|
import { getConnectorVerificationTargets } from "../../serve/connectors";
|
|
@@ -64,9 +65,56 @@ export type StatusResult =
|
|
|
64
65
|
contentTypeBoost: ContentTypeBoostStatus;
|
|
65
66
|
memory: MemoryStatus;
|
|
66
67
|
backgroundIssues: ResidentBackgroundIssue[];
|
|
68
|
+
/** Files whose latest conversion stopped at the per-file budget. */
|
|
69
|
+
budgetStops: BudgetStop[];
|
|
67
70
|
}
|
|
68
71
|
| { success: false; error: string };
|
|
69
72
|
|
|
73
|
+
/** A file left unindexed because its conversion overran its budget. */
|
|
74
|
+
export interface BudgetStop {
|
|
75
|
+
collection: string;
|
|
76
|
+
relPath: string;
|
|
77
|
+
code: string;
|
|
78
|
+
message: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Most recent ingest errors inspected for budget stops. */
|
|
82
|
+
const BUDGET_STOP_SCAN_LIMIT = 100;
|
|
83
|
+
/** Most budget stops listed in status output. */
|
|
84
|
+
const MAX_BUDGET_STOPS = 10;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Files whose latest indexing attempt stopped at the conversion budget, from
|
|
88
|
+
* the recent error log, kept only while the document still carries that code.
|
|
89
|
+
*/
|
|
90
|
+
async function collectBudgetStops(store: SqliteAdapter): Promise<BudgetStop[]> {
|
|
91
|
+
const recent = await store.getRecentErrors(BUDGET_STOP_SCAN_LIMIT);
|
|
92
|
+
if (!recent.ok) return [];
|
|
93
|
+
const stops: BudgetStop[] = [];
|
|
94
|
+
const seen = new Set<string>();
|
|
95
|
+
for (const row of recent.value) {
|
|
96
|
+
const key = `${row.collection}\u0000${row.relPath}`;
|
|
97
|
+
if (seen.has(key)) continue;
|
|
98
|
+
seen.add(key);
|
|
99
|
+
if (!BUDGET_ERROR_CODES.has(row.code)) continue;
|
|
100
|
+
const document = await store.getDocument(row.collection, row.relPath);
|
|
101
|
+
if (
|
|
102
|
+
document.ok &&
|
|
103
|
+
document.value?.active &&
|
|
104
|
+
document.value.lastErrorCode === row.code
|
|
105
|
+
) {
|
|
106
|
+
stops.push({
|
|
107
|
+
collection: row.collection,
|
|
108
|
+
relPath: row.relPath,
|
|
109
|
+
code: row.code,
|
|
110
|
+
message: row.message,
|
|
111
|
+
});
|
|
112
|
+
if (stops.length >= MAX_BUDGET_STOPS) break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return stops;
|
|
116
|
+
}
|
|
117
|
+
|
|
70
118
|
/** A background issue reported by (or about) a running detached resident. */
|
|
71
119
|
export type ResidentBackgroundIssue = BackgroundIssue & {
|
|
72
120
|
process: "serve" | "daemon";
|
|
@@ -368,6 +416,7 @@ export async function status(
|
|
|
368
416
|
contentTypeBoost: buildContentTypeBoostStatus(config.contentTypes ?? []),
|
|
369
417
|
memory: await buildMemoryStatus(store, config.collections),
|
|
370
418
|
backgroundIssues: await collectResidentIssues(),
|
|
419
|
+
budgetStops: await collectBudgetStops(store),
|
|
371
420
|
};
|
|
372
421
|
} finally {
|
|
373
422
|
await store.close();
|
|
@@ -439,6 +488,10 @@ export function formatStatus(
|
|
|
439
488
|
(issue) =>
|
|
440
489
|
` ${issue.process} (pid ${issue.pid}): ${formatBackgroundIssue(issue)}`
|
|
441
490
|
);
|
|
491
|
+
const stopLines = result.budgetStops.map(
|
|
492
|
+
(stop) =>
|
|
493
|
+
` ${stop.collection}/${stop.relPath} [${stop.code}] ${stop.message}`
|
|
494
|
+
);
|
|
442
495
|
return [
|
|
443
496
|
formatTerminal(
|
|
444
497
|
result.status,
|
|
@@ -446,6 +499,13 @@ export function formatStatus(
|
|
|
446
499
|
result.contentTypeBoost,
|
|
447
500
|
result.memory
|
|
448
501
|
),
|
|
502
|
+
...(stopLines.length
|
|
503
|
+
? [
|
|
504
|
+
"",
|
|
505
|
+
"Stopped at conversion budget (retried by the next update):",
|
|
506
|
+
...stopLines,
|
|
507
|
+
]
|
|
508
|
+
: []),
|
|
449
509
|
...(issueLines.length ? ["", "Background issues:", ...issueLines] : []),
|
|
450
510
|
].join("\n");
|
|
451
511
|
}
|
|
@@ -15,7 +15,11 @@ import {
|
|
|
15
15
|
type SyncResult,
|
|
16
16
|
withContentTypeRules,
|
|
17
17
|
} from "../../ingestion";
|
|
18
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
formatSyncResultLines,
|
|
20
|
+
initStore,
|
|
21
|
+
reportSlowConversion,
|
|
22
|
+
} from "./shared";
|
|
19
23
|
|
|
20
24
|
/**
|
|
21
25
|
* Options for update command.
|
|
@@ -66,6 +70,7 @@ export async function update(
|
|
|
66
70
|
{
|
|
67
71
|
gitPull: options.gitPull,
|
|
68
72
|
runUpdateCmd: true,
|
|
73
|
+
onSlowConversion: reportSlowConversion,
|
|
69
74
|
},
|
|
70
75
|
config
|
|
71
76
|
)
|
package/src/cli/commands/vec.ts
CHANGED
|
@@ -13,10 +13,16 @@ import {
|
|
|
13
13
|
createVectorIndexPort,
|
|
14
14
|
createVectorStatsPort,
|
|
15
15
|
} from "../../store/vector";
|
|
16
|
+
import {
|
|
17
|
+
rebuildPartitionIndex,
|
|
18
|
+
syncPartitionIndex,
|
|
19
|
+
} from "../../store/vector/partition-index";
|
|
16
20
|
import {
|
|
17
21
|
dropVectorPartition,
|
|
22
|
+
storedVectorPartition,
|
|
18
23
|
type VectorPartitionStatus,
|
|
19
24
|
} from "../../store/vector/status";
|
|
25
|
+
import { loadSqliteVec } from "../../store/vector/variants";
|
|
20
26
|
|
|
21
27
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
22
28
|
// Types
|
|
@@ -76,7 +82,9 @@ function inferDimensions(
|
|
|
76
82
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
77
83
|
|
|
78
84
|
/**
|
|
79
|
-
* Sync vec0 index with
|
|
85
|
+
* Sync the vec0 index with stored vectors (add missing, remove orphans): the
|
|
86
|
+
* active vector partition's variants, or legacy `content_vectors` on an
|
|
87
|
+
* index that never activated a partition.
|
|
80
88
|
*/
|
|
81
89
|
export async function vecSync(
|
|
82
90
|
options: VecOptions = {}
|
|
@@ -112,6 +120,21 @@ export async function vecSync(
|
|
|
112
120
|
try {
|
|
113
121
|
const db = store.getRawDb();
|
|
114
122
|
|
|
123
|
+
const partition = storedVectorPartition(db, modelUri);
|
|
124
|
+
if (partition) {
|
|
125
|
+
if (!(await loadSqliteVec(db))) {
|
|
126
|
+
return {
|
|
127
|
+
success: false,
|
|
128
|
+
error: "sqlite-vec not available. Cannot sync index.",
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
return {
|
|
132
|
+
success: true,
|
|
133
|
+
...syncPartitionIndex(db, partition),
|
|
134
|
+
model: modelUri,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
115
138
|
// Infer dimensions from stored vectors for this model
|
|
116
139
|
const dimensions = inferDimensions(db, modelUri);
|
|
117
140
|
if (dimensions === null) {
|
|
@@ -154,7 +177,9 @@ export async function vecSync(
|
|
|
154
177
|
}
|
|
155
178
|
|
|
156
179
|
/**
|
|
157
|
-
* Rebuild vec0 index
|
|
180
|
+
* Rebuild the vec0 index (drop + recreate + repopulate) from the active
|
|
181
|
+
* vector partition's variants, or from legacy `content_vectors` on an index
|
|
182
|
+
* that never activated a partition.
|
|
158
183
|
*/
|
|
159
184
|
export async function vecRebuild(
|
|
160
185
|
options: VecOptions = {}
|
|
@@ -190,6 +215,21 @@ export async function vecRebuild(
|
|
|
190
215
|
try {
|
|
191
216
|
const db = store.getRawDb();
|
|
192
217
|
|
|
218
|
+
const partition = storedVectorPartition(db, modelUri);
|
|
219
|
+
if (partition) {
|
|
220
|
+
if (!(await loadSqliteVec(db))) {
|
|
221
|
+
return {
|
|
222
|
+
success: false,
|
|
223
|
+
error: "sqlite-vec not available. Cannot rebuild index.",
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
return {
|
|
227
|
+
success: true,
|
|
228
|
+
count: rebuildPartitionIndex(db, partition),
|
|
229
|
+
model: modelUri,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
193
233
|
// Infer dimensions from stored vectors for this model
|
|
194
234
|
const dimensions = inferDimensions(db, modelUri);
|
|
195
235
|
if (dimensions === null) {
|