@gmickel/gno 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +4 -2
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.1.zip → gno-browser-clipper-v2.8.3.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +9 -1
- package/spec/cli.md +81 -8
- package/spec/mcp.md +5 -6
- package/src/cli/commands/daemon.ts +7 -2
- package/src/cli/commands/doctor.ts +42 -24
- package/src/cli/commands/index-cmd.ts +10 -2
- package/src/cli/commands/links.ts +5 -22
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/status.ts +60 -0
- package/src/cli/commands/update.ts +6 -1
- package/src/cli/commands/vec.ts +42 -2
- package/src/cli/program.ts +2 -2
- package/src/config/types.ts +19 -0
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
- package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
- package/src/converters/adapters/xlsx/adapter.ts +132 -0
- package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
- package/src/converters/native/markdown.ts +3 -1
- package/src/converters/registry.ts +62 -6
- package/src/converters/versions.ts +2 -0
- package/src/core/audit-freshness.ts +1 -1
- package/src/core/audit-links.ts +2 -1
- package/src/core/audit-outside-index.ts +91 -18
- package/src/core/change-diff.ts +18 -6
- package/src/core/link-destination-parse.ts +21 -4
- package/src/core/link-inventory-markdown.ts +23 -3
- package/src/core/link-inventory-opaque.ts +3 -1
- package/src/core/links.ts +74 -16
- package/src/core/network-boundary-inventory.ts +18 -0
- package/src/index.ts +10 -0
- package/src/ingestion/document-metadata.ts +256 -0
- package/src/ingestion/file-child-env.ts +9 -0
- package/src/ingestion/file-child.ts +106 -0
- package/src/ingestion/file-processor.ts +546 -0
- package/src/ingestion/frontmatter.ts +6 -2
- package/src/ingestion/index.ts +1 -0
- package/src/ingestion/prepare-convert.ts +32 -0
- package/src/ingestion/prepare-file.ts +272 -0
- package/src/ingestion/prepare-worker.ts +43 -0
- package/src/ingestion/record-container.ts +7 -2
- package/src/ingestion/strip.ts +149 -133
- package/src/ingestion/sync-options.ts +7 -1
- package/src/ingestion/sync.ts +208 -385
- package/src/ingestion/types.ts +18 -0
- package/src/llm/native-worker/embedding-identity.ts +15 -3
- package/src/mcp/http-transport.ts +11 -0
- package/src/mcp/tools/links.ts +10 -38
- package/src/serve/resident-request.ts +19 -0
- package/src/serve/resident-runtime.ts +84 -8
- package/src/serve/routes/links.ts +6 -24
- package/src/serve/server.ts +3 -2
- package/src/sessions/config-refresh.ts +3 -2
- package/src/store/sqlite/graph-similarity.ts +5 -13
- package/src/store/vector/partition-index.ts +100 -0
- package/src/store/vector/stored-vectors.ts +20 -15
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.1.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
|
|
|
139
139
|
|
|
140
140
|
<!-- public-truth:current-version -->
|
|
141
141
|
|
|
142
|
-
> Current source version: **v2.8.
|
|
142
|
+
> Current source version: **v2.8.3**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
143
143
|
|
|
144
144
|
<!-- /public-truth -->
|
|
145
145
|
|
package/assets/skill/SKILL.md
CHANGED
|
@@ -356,8 +356,10 @@ rewrite, persist findings, judge factual truth, or replace retrieval.
|
|
|
356
356
|
finding. Link findings carry `referenceKind`, `resolutionStatus`, and
|
|
357
357
|
`resolvedScope` in their evidence detail; ambiguous vault links list the tied
|
|
358
358
|
`candidates`. `links.outside-index` `info` findings are link targets that
|
|
359
|
-
exist in the vault but are not indexed (attachments,
|
|
360
|
-
folders); Obsidian resolves them, so do
|
|
359
|
+
exist in the vault but are not indexed (attachments, files a Markdown link
|
|
360
|
+
names by path, excluded or unindexed folders); Obsidian resolves them, so do
|
|
361
|
+
not report them as broken links. Links are read only from Markdown notes,
|
|
362
|
+
never from code spans, code blocks, or non-Markdown files.
|
|
361
363
|
Report `truncation.snapshotTruncated` as "totals cover the bounded snapshot,
|
|
362
364
|
not the whole index".
|
|
363
365
|
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
7fe15ce08a24f4dae4ac6694f62bb26d94bbc47d6abc09826f29267ace10558d gno-browser-clipper-v2.8.3.zip
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gmickel/gno",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.3",
|
|
4
4
|
"description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"embeddings",
|
|
@@ -197,7 +197,15 @@
|
|
|
197
197
|
"mammoth": "1.12.2",
|
|
198
198
|
"mdast-util-from-markdown": "2.0.3",
|
|
199
199
|
"mdast-util-gfm": "3.1.0",
|
|
200
|
+
"mdast-util-gfm-autolink-literal": "2.0.1",
|
|
201
|
+
"mdast-util-gfm-footnote": "2.1.0",
|
|
202
|
+
"mdast-util-gfm-strikethrough": "2.0.0",
|
|
203
|
+
"mdast-util-gfm-task-list-item": "2.0.0",
|
|
200
204
|
"micromark-extension-gfm": "3.0.0",
|
|
205
|
+
"micromark-extension-gfm-autolink-literal": "2.1.0",
|
|
206
|
+
"micromark-extension-gfm-footnote": "2.1.0",
|
|
207
|
+
"micromark-extension-gfm-strikethrough": "2.1.0",
|
|
208
|
+
"micromark-extension-gfm-task-list-item": "2.1.0",
|
|
201
209
|
"mime-types": "2.1.35",
|
|
202
210
|
"minimatch": "10.2.6",
|
|
203
211
|
"nanoid": "6.0.1",
|
package/spec/cli.md
CHANGED
|
@@ -965,18 +965,32 @@ Orphans stay "no incoming or outgoing resolved links", with connectivity drawn
|
|
|
965
965
|
from the whole index even when the audited documents are scoped; a tied link
|
|
966
966
|
connects nothing.
|
|
967
967
|
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
968
|
+
Links are parsed only from documents converted by the Markdown converter
|
|
969
|
+
(`native/markdown`); plain text, source code, data files and converted
|
|
970
|
+
documents store no links and produce no link findings, while staying
|
|
971
|
+
searchable. The parser skips frontmatter, HTML comments, and code as a
|
|
972
|
+
CommonMark + GFM parser reads it: inline code spans, fenced code blocks and
|
|
973
|
+
indented code blocks, inside blockquotes and list items too (an indented list
|
|
974
|
+
paragraph is not code). A link outside code on the same line still counts.
|
|
975
|
+
|
|
976
|
+
A plain wiki link, embed, or relative Markdown link from a document in a link
|
|
977
|
+
workspace whose target is not an indexed document but exists as a file inside
|
|
978
|
+
the workspace root (a non-Markdown attachment, a script beside the note, a
|
|
979
|
+
note in an unindexed folder, a note excluded by a collection pattern) is not
|
|
980
|
+
reported by `links.local-targets`. It is an `info`
|
|
972
981
|
finding of `links.outside-index` (evidence kind `outside-index-target`,
|
|
973
982
|
`resolutionStatus: "outside-index"`). That rule stays `pass`, so these
|
|
974
983
|
findings never change the exit code. The check is existence-only: each
|
|
975
984
|
involved workspace root is listed once per run (hidden files and folders
|
|
976
985
|
skipped, symlinked folders not descended, a symlink counted only when it
|
|
977
986
|
resolves to a regular file inside the workspace, at most 200,000 files), no
|
|
978
|
-
file is opened, and no graph edge is created.
|
|
987
|
+
file is opened, and no graph edge is created. Wiki targets match files by the workspace
|
|
979
988
|
link rules; a non-Markdown target needs its extension, `.md` is optional. A
|
|
989
|
+
Markdown link matches only the file at its resolved path (NFC,
|
|
990
|
+
case-insensitive); as for wiki links, `.md` is optional, so `[x](Note)` matches
|
|
991
|
+
an unindexed `Note.md`. An indexed `Note.md` (compared under the same NFC,
|
|
992
|
+
Unicode-lowercase key) is not outside the index, and Markdown link resolution
|
|
993
|
+
itself stays exact-path, so that link remains in `links.local-targets`. A
|
|
980
994
|
target missing from the listing stays unresolved; an unreadable folder or the
|
|
981
995
|
file bound makes the listing incomplete, which the `links.outside-index` rule
|
|
982
996
|
message states once.
|
|
@@ -984,7 +998,12 @@ message states once.
|
|
|
984
998
|
The wiki link parser reads Obsidian's table-escaped alias `[[Note\|Alias]]`
|
|
985
999
|
as target `Note` with alias `Alias`. Markdown link text may contain balanced
|
|
986
1000
|
square brackets (`[see [1]](note.md)`); a destination containing a square
|
|
987
|
-
bracket is not a link.
|
|
1001
|
+
bracket is not a link. Markdown links follow CommonMark, as Obsidian renders
|
|
1002
|
+
them: a `[` escaped with a backslash (`\[clause](see section 4)`) opens no
|
|
1003
|
+
link, and an inline destination with an unescaped space is plain text unless
|
|
1004
|
+
it is wrapped in angle brackets. `[x](<my note.md>)` and `[x](my%20note.md)`
|
|
1005
|
+
both link to `my note.md`; a link title (`[x](note.md "Title")`) is not part
|
|
1006
|
+
of the target.
|
|
988
1007
|
|
|
989
1008
|
The JSON contract is versioned as `gno://schemas/audit-report@1.0`. Finding IDs
|
|
990
1009
|
are stable SHA-256 identities derived from rule, normalized subject/location,
|
|
@@ -1461,6 +1480,33 @@ individual virtual document. The closed `record.adapter` identity contains the
|
|
|
1461
1480
|
adapter ID, version, and configuration fingerprint. `get` and `multi-get`
|
|
1462
1481
|
preserve the same fields.
|
|
1463
1482
|
|
|
1483
|
+
Every file is indexed under the per-file budget `conversion.timeoutMs`
|
|
1484
|
+
(default 60000) and `conversion.maxMemoryMb` (default half of physical memory,
|
|
1485
|
+
at least 2048; process resident memory). Conversion and every post-conversion
|
|
1486
|
+
phase (metadata, code regions, change-journal structure, chunking, link
|
|
1487
|
+
extraction) run in one worker per process, one file at a time; the worker is
|
|
1488
|
+
terminated at either limit, mid-phase, and replaced for the next file.
|
|
1489
|
+
Database reads and writes stay on the main thread. A standalone compiled
|
|
1490
|
+
executable, which cannot start the worker, runs the phases in a child process
|
|
1491
|
+
of itself (internal environment flag) under the same limits; the child's
|
|
1492
|
+
resident memory counts toward `conversion.maxMemoryMb` (Linux `/proc`, `ps`
|
|
1493
|
+
elsewhere; time only on Windows). A processor that cannot start fails the file
|
|
1494
|
+
with `ISOLATION_UNAVAILABLE`; nothing is prepared without the budget. The
|
|
1495
|
+
child is SIGKILLed on process exit, SIGTERM, SIGINT (via the CLI's exit
|
|
1496
|
+
path) and resident shutdown; on Linux it also carries a parent-death signal,
|
|
1497
|
+
so it dies with a SIGKILLed parent, and elsewhere it exits at its next step
|
|
1498
|
+
once its parent is gone.
|
|
1499
|
+
A file over either limit is recorded with error code `TIMEOUT` or
|
|
1500
|
+
`MEMORY_LIMIT` (the message names the running phase), stays unindexed, and is
|
|
1501
|
+
retried by the next sync; the rest of the run continues. `update` and `index` always print such a
|
|
1502
|
+
file in terminal output as `[CODE] <relPath>: <message>` (other per-file
|
|
1503
|
+
errors stay `--verbose` only), and write
|
|
1504
|
+
`Still converting <collection>/<relPath> after <n>s (budget <m>s)` to stderr
|
|
1505
|
+
once a file has been processing for 10 seconds or half its budget, whichever
|
|
1506
|
+
is sooner.
|
|
1507
|
+
`gno status` lists stopped files under "Stopped at conversion budget" in
|
|
1508
|
+
terminal output (JSON output is unchanged).
|
|
1509
|
+
|
|
1464
1510
|
File/export adapter iteration has a 60-second deadline in addition to the byte,
|
|
1465
1511
|
record, metadata, and total-character caps. A partial import that retains valid
|
|
1466
1512
|
siblings exits successfully but is never silent: terminal output includes the
|
|
@@ -2965,7 +3011,11 @@ misconfigured `findings` block (see [Daemon Mode](../docs/DAEMON.md#scheduled-fi
|
|
|
2965
3011
|
The `embedding-fingerprint` check is additive doctor-only diagnostics. It uses
|
|
2966
3012
|
the active embed model and stored vector dimensions to report the current
|
|
2967
3013
|
freshness fingerprint, pending/stale chunks, legacy empty-fingerprint vectors,
|
|
2968
|
-
and stored fingerprint groups.
|
|
3014
|
+
and stored fingerprint groups. On an index with an activated vector partition
|
|
3015
|
+
for the embed model (every index `gno embed` wrote since 2.7), the groups are
|
|
3016
|
+
that partition with its current chunk count and there are no legacy vectors;
|
|
3017
|
+
legacy `content_vectors` groups are read only before any partition activates.
|
|
3018
|
+
Stale, legacy, and mixed groups are warnings;
|
|
2969
3019
|
recover with `gno embed`, or `gno embed --force` if vectors still look stale.
|
|
2970
3020
|
|
|
2971
3021
|
The additive `activation` object uses the same contract as `gno status` and
|
|
@@ -3977,7 +4027,10 @@ gno similar <doc> [-n, --limit <num>] [--threshold <num>] [--cross-collection] [
|
|
|
3977
4027
|
|
|
3978
4028
|
- Finds documents semantically similar to the source document
|
|
3979
4029
|
- Requires embeddings to be generated (`gno embed`)
|
|
3980
|
-
-
|
|
4030
|
+
- Source vector: the stored vector of the document's first chunk (the
|
|
4031
|
+
lowest-seq chunk whose vector is current), read from the active vector
|
|
4032
|
+
partition. `gno_similar` (MCP), `GET /api/doc/:id/similar` (REST) and graph
|
|
4033
|
+
similarity edges use the same rule, so scores match across surfaces
|
|
3981
4034
|
- By default, limits results to same collection
|
|
3982
4035
|
|
|
3983
4036
|
**Output (JSON):**
|
|
@@ -4192,6 +4245,9 @@ gno changes --follow --jsonl [--cursor <cursor>] [--collection <name>]
|
|
|
4192
4245
|
old/new identity and hash snapshots, normalized structural deltas, pagination,
|
|
4193
4246
|
cursor-expiry, and retention-truncation disclosure.
|
|
4194
4247
|
- The journal never returns source bodies.
|
|
4248
|
+
- Link additions and removals in `structureDelta.links` are recorded only for
|
|
4249
|
+
Markdown sources, matching link extraction; content changes of other
|
|
4250
|
+
documents (code, text, converted files, records) are still journaled.
|
|
4195
4251
|
|
|
4196
4252
|
**Follow mode (`--follow --jsonl`)** streams journal events as they land and is
|
|
4197
4253
|
the durable automation input for consumers that resume across restarts.
|
|
@@ -4433,6 +4489,21 @@ gno serve --detach --stop
|
|
|
4433
4489
|
|
|
4434
4490
|
---
|
|
4435
4491
|
|
|
4492
|
+
#### Config reload while running
|
|
4493
|
+
|
|
4494
|
+
A running `gno serve` or `gno daemon` re-reads its config file before each
|
|
4495
|
+
request (REST, Web UI, HTTP MCP, daemon status) and every 2 seconds, and adopts
|
|
4496
|
+
changes to `collections` and `contexts` made by the CLI (`gno collection
|
|
4497
|
+
add/remove`) or by hand: status, MCP `gno_status`, the watcher and search scope
|
|
4498
|
+
follow without a restart. A newly added collection is watched from then on; its
|
|
4499
|
+
existing files are indexed by `gno update` (or the next start's initial sync).
|
|
4500
|
+
An adoption that changes the collection set or policies moves the egress policy
|
|
4501
|
+
epoch as a REST collection change does. The checks are the sessions config
|
|
4502
|
+
refresh's: an unreadable config file answers an error (`500`, sessions code
|
|
4503
|
+
`SESSIONS_RUNTIME_FAILURE`), never the stale config, and a config bound to
|
|
4504
|
+
another index is refused. Other settings (models, gateway, sessions sources
|
|
4505
|
+
outside the sessions routes) keep their restart behaviour.
|
|
4506
|
+
|
|
4436
4507
|
### gno daemon
|
|
4437
4508
|
|
|
4438
4509
|
Start a headless long-running watcher process for continuous indexing.
|
|
@@ -4472,6 +4543,8 @@ is blocked.
|
|
|
4472
4543
|
|
|
4473
4544
|
- Opens DB once at startup
|
|
4474
4545
|
- Loads config and requires at least one configured collection
|
|
4546
|
+
- Follows collection edits to the config file while running (see
|
|
4547
|
+
[Config reload](#config-reload-while-running))
|
|
4475
4548
|
- Starts the same watcher + embed scheduler used by `gno serve`
|
|
4476
4549
|
- Exact contained eligible paths always use targeted content-hash sync.
|
|
4477
4550
|
Ambiguous temp/directory/missing-name/recursive-delete events use bounded,
|
package/spec/mcp.md
CHANGED
|
@@ -2188,12 +2188,11 @@ Find semantically similar documents using vector embeddings.
|
|
|
2188
2188
|
|
|
2189
2189
|
**Algorithm:**
|
|
2190
2190
|
|
|
2191
|
-
1.
|
|
2192
|
-
2.
|
|
2193
|
-
3.
|
|
2194
|
-
4.
|
|
2195
|
-
5.
|
|
2196
|
-
6. Return top N similar documents with scores
|
|
2191
|
+
1. Take the stored vector of the source document's first chunk (the lowest-seq chunk whose vector is current) from the active vector partition (legacy `content_vectors` only before any partition activates); no model is loaded. This is the same source vector `gno similar`, `GET /api/doc/:id/similar` and graph similarity edges use, so scores match across surfaces
|
|
2192
|
+
2. Normalize it to unit length
|
|
2193
|
+
3. Search for nearest neighbors using sqlite-vec
|
|
2194
|
+
4. Exclude self and filter by collection if not crossCollection
|
|
2195
|
+
5. Return top N similar documents with scores
|
|
2197
2196
|
|
|
2198
2197
|
**Errors:**
|
|
2199
2198
|
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
resolveHttpGatewayConfig,
|
|
17
17
|
} from "../../mcp/http-security";
|
|
18
18
|
import { startBackgroundRuntime } from "../../serve/background-runtime";
|
|
19
|
+
import { refreshResidentConfig } from "../../serve/resident-request";
|
|
19
20
|
import { handleResidentStatus, handleStatus } from "../../serve/routes/api";
|
|
20
21
|
import { createMcpHttpGateway } from "../../serve/routes/mcp";
|
|
21
22
|
import { SessionsError } from "../../sessions/types";
|
|
@@ -106,6 +107,8 @@ export const authorizeDaemonStatus = async (
|
|
|
106
107
|
): Promise<Response> => {
|
|
107
108
|
const authorization = await gateway.security.authorize(request, server);
|
|
108
109
|
if (!authorization.ok) return authorization.response;
|
|
110
|
+
const refreshFailure = await refreshResidentConfig(runtime);
|
|
111
|
+
if (refreshFailure) return refreshFailure;
|
|
109
112
|
try {
|
|
110
113
|
enforceCollectionEgress({
|
|
111
114
|
collections: runtime.config.collections,
|
|
@@ -132,13 +135,15 @@ export const authorizeDaemonStatus = async (
|
|
|
132
135
|
}
|
|
133
136
|
};
|
|
134
137
|
|
|
135
|
-
export function handleDaemonAppStatus(
|
|
138
|
+
export async function handleDaemonAppStatus(
|
|
136
139
|
runtime: ResidentRuntime,
|
|
137
140
|
host: string
|
|
138
|
-
): Promise<Response>
|
|
141
|
+
): Promise<Response> {
|
|
139
142
|
if (!isHttpGatewayLoopbackBind(host)) {
|
|
140
143
|
return new Response(null, { status: 404 });
|
|
141
144
|
}
|
|
145
|
+
const refreshFailure = await refreshResidentConfig(runtime);
|
|
146
|
+
if (refreshFailure) return refreshFailure;
|
|
142
147
|
return handleStatus(runtime.ctxHolder.current, {
|
|
143
148
|
getResidentStatus: () => runtime.getStatus(),
|
|
144
149
|
});
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
resolveFindingsSchedule,
|
|
32
32
|
} from "../../core/findings-run-state";
|
|
33
33
|
import { formatVectorPartitionLines } from "../../core/vector-partition-status";
|
|
34
|
+
import { getEmbeddingFingerprint } from "../../embed/fingerprint";
|
|
34
35
|
import { getCodeChunkingStatus } from "../../ingestion/chunker";
|
|
35
36
|
import { ModelCache } from "../../llm/cache";
|
|
36
37
|
import { getActivePreset, resolveModelUri } from "../../llm/registry";
|
|
@@ -42,6 +43,8 @@ import {
|
|
|
42
43
|
getLoadAttempts,
|
|
43
44
|
} from "../../store/sqlite/setup";
|
|
44
45
|
import { getStoredEmbeddingFingerprint } from "../../store/vector/freshness";
|
|
46
|
+
import { currentOwnerCount } from "../../store/vector/runtime-compat";
|
|
47
|
+
import { storedVectorPartition } from "../../store/vector/status";
|
|
45
48
|
import {
|
|
46
49
|
buildDoctorActivation,
|
|
47
50
|
checkConnectorActivation,
|
|
@@ -319,10 +322,14 @@ async function checkEmbeddingFingerprints(
|
|
|
319
322
|
];
|
|
320
323
|
}
|
|
321
324
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
325
|
+
// An index embedded since 2.7 keeps its vectors in the active partition;
|
|
326
|
+
// legacy `content_vectors` count only before any partition activates.
|
|
327
|
+
const partition = storedVectorPartition(db, model);
|
|
328
|
+
const legacyChunks = partition
|
|
329
|
+
? 0
|
|
330
|
+
: (db
|
|
331
|
+
.query<{ count: number }, [string]>(
|
|
332
|
+
`
|
|
326
333
|
SELECT COUNT(*) as count
|
|
327
334
|
FROM content_vectors v
|
|
328
335
|
JOIN content_chunks c
|
|
@@ -336,12 +343,23 @@ async function checkEmbeddingFingerprints(
|
|
|
336
343
|
AND d.active = 1
|
|
337
344
|
)
|
|
338
345
|
`
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const groups =
|
|
343
|
-
|
|
344
|
-
|
|
346
|
+
)
|
|
347
|
+
.get(model)?.count ?? 0);
|
|
348
|
+
|
|
349
|
+
const groups = partition
|
|
350
|
+
? [
|
|
351
|
+
{
|
|
352
|
+
model,
|
|
353
|
+
fingerprint: getEmbeddingFingerprint({
|
|
354
|
+
modelUri: model,
|
|
355
|
+
dimensions: partition.dimensions,
|
|
356
|
+
}),
|
|
357
|
+
count: currentOwnerCount(db, partition.partitionId),
|
|
358
|
+
},
|
|
359
|
+
]
|
|
360
|
+
: db
|
|
361
|
+
.query<{ model: string; fingerprint: string; count: number }, []>(
|
|
362
|
+
`
|
|
345
363
|
SELECT
|
|
346
364
|
v.model as model,
|
|
347
365
|
v.embed_fingerprint as fingerprint,
|
|
@@ -358,18 +376,18 @@ async function checkEmbeddingFingerprints(
|
|
|
358
376
|
GROUP BY v.model, v.embed_fingerprint
|
|
359
377
|
ORDER BY count DESC, v.model ASC, v.embed_fingerprint ASC
|
|
360
378
|
`
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
const mixedGroups =
|
|
379
|
+
)
|
|
380
|
+
.all();
|
|
381
|
+
const fingerprintGroups = groups.map((group) => ({
|
|
382
|
+
model: group.model,
|
|
383
|
+
fingerprint: group.fingerprint,
|
|
384
|
+
count: group.count,
|
|
385
|
+
current:
|
|
386
|
+
group.model === model && group.fingerprint === currentFingerprint,
|
|
387
|
+
legacy: group.fingerprint === "",
|
|
388
|
+
}));
|
|
389
|
+
|
|
390
|
+
const mixedGroups = fingerprintGroups.length;
|
|
373
391
|
const pendingChunks = statusResult.value.embeddingBacklog;
|
|
374
392
|
const hasWarnings =
|
|
375
393
|
pendingChunks > 0 || legacyChunks > 0 || mixedGroups > 1;
|
|
@@ -380,7 +398,7 @@ async function checkEmbeddingFingerprints(
|
|
|
380
398
|
pendingChunks,
|
|
381
399
|
legacyChunks,
|
|
382
400
|
mixedGroups,
|
|
383
|
-
groups,
|
|
401
|
+
groups: fingerprintGroups,
|
|
384
402
|
};
|
|
385
403
|
|
|
386
404
|
const message =
|
|
@@ -392,7 +410,7 @@ async function checkEmbeddingFingerprints(
|
|
|
392
410
|
if (hasWarnings) {
|
|
393
411
|
details.push("Run: gno embed");
|
|
394
412
|
details.push("If vectors still look stale, run: gno embed --force");
|
|
395
|
-
for (const group of
|
|
413
|
+
for (const group of fingerprintGroups) {
|
|
396
414
|
const label = describeFingerprintGroup(group);
|
|
397
415
|
const fingerprint = group.legacy
|
|
398
416
|
? "(empty)"
|
|
@@ -28,7 +28,11 @@ import {
|
|
|
28
28
|
withContentTypeRules,
|
|
29
29
|
} from "../../ingestion";
|
|
30
30
|
import { type EmbedResult, embedStageOutcome } from "./embed";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
formatSyncResultLines,
|
|
33
|
+
initStore,
|
|
34
|
+
reportSlowConversion,
|
|
35
|
+
} from "./shared";
|
|
32
36
|
|
|
33
37
|
/**
|
|
34
38
|
* Options for index command.
|
|
@@ -202,7 +206,11 @@ export async function index(options: IndexOptions = {}): Promise<IndexResult> {
|
|
|
202
206
|
collections,
|
|
203
207
|
store,
|
|
204
208
|
withContentTypeRules(
|
|
205
|
-
{
|
|
209
|
+
{
|
|
210
|
+
gitPull: options.gitPull,
|
|
211
|
+
runUpdateCmd: true,
|
|
212
|
+
onSlowConversion: reportSlowConversion,
|
|
213
|
+
},
|
|
206
214
|
config
|
|
207
215
|
)
|
|
208
216
|
);
|
|
@@ -587,21 +587,17 @@ export async function similar(
|
|
|
587
587
|
}
|
|
588
588
|
const db = store.getRawDb();
|
|
589
589
|
|
|
590
|
-
//
|
|
590
|
+
// Similarity source vector (first chunk, unit length), active partition
|
|
591
591
|
const {
|
|
592
|
-
|
|
592
|
+
readSimilaritySourceVectors,
|
|
593
593
|
resolveStoredVectorSource,
|
|
594
594
|
similarityHitDocuments,
|
|
595
595
|
storedVectorSearchOptions,
|
|
596
596
|
} = await import("../../store/vector/stored-vectors.js");
|
|
597
597
|
const source = resolveStoredVectorSource(db, modelPreset.embed);
|
|
598
|
-
const
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
source,
|
|
602
|
-
[{ id: doc.id, mirrorHash: doc.mirrorHash }],
|
|
603
|
-
{ firstChunkOnly: true }
|
|
604
|
-
).get(doc.id) ?? [];
|
|
598
|
+
const embedding = readSimilaritySourceVectors(db, source, [
|
|
599
|
+
{ id: doc.id, mirrorHash: doc.mirrorHash },
|
|
600
|
+
]).get(doc.id);
|
|
605
601
|
|
|
606
602
|
if (!embedding) {
|
|
607
603
|
return {
|
|
@@ -610,20 +606,7 @@ export async function similar(
|
|
|
610
606
|
isValidation: true,
|
|
611
607
|
};
|
|
612
608
|
}
|
|
613
|
-
|
|
614
|
-
// Normalize embedding for cosine similarity
|
|
615
609
|
const dimensions = embedding.length;
|
|
616
|
-
let norm = 0;
|
|
617
|
-
for (let i = 0; i < dimensions; i++) {
|
|
618
|
-
const val = embedding[i] ?? 0;
|
|
619
|
-
norm += val * val;
|
|
620
|
-
}
|
|
621
|
-
norm = Math.sqrt(norm);
|
|
622
|
-
if (norm > 0) {
|
|
623
|
-
for (let i = 0; i < dimensions; i++) {
|
|
624
|
-
embedding[i] = (embedding[i] ?? 0) / norm;
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
610
|
|
|
628
611
|
// Create vector index port for search
|
|
629
612
|
const { createVectorIndexPort } =
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { Collection, Config } from "../../config/types";
|
|
9
|
-
import type { SyncResult } from "../../ingestion";
|
|
9
|
+
import type { SlowConversionEvent, SyncResult } from "../../ingestion";
|
|
10
10
|
import type { SearchResults } from "../../pipeline/types";
|
|
11
11
|
|
|
12
12
|
import {
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
loadConfig,
|
|
21
21
|
writeConfigWarningsToStderr,
|
|
22
22
|
} from "../../config";
|
|
23
|
+
import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
|
|
23
24
|
import { SqliteAdapter } from "../../store/sqlite/adapter";
|
|
24
25
|
import { assertCliSessionBinding } from "../session-binding";
|
|
25
26
|
|
|
@@ -164,6 +165,21 @@ export function decorateSearchResultsForIndex(
|
|
|
164
165
|
};
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Name a file whose conversion is still running past the notice threshold,
|
|
170
|
+
* so an operator can see which file a slow run is working on (stderr, so
|
|
171
|
+
* JSON output on stdout stays parseable).
|
|
172
|
+
*/
|
|
173
|
+
export function reportSlowConversion(event: SlowConversionEvent): void {
|
|
174
|
+
process.stderr.write(
|
|
175
|
+
`Still converting ${event.collection}/${event.relPath} after ${formatSeconds(event.elapsedMs)} (budget ${formatSeconds(event.budgetMs)})\n`
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Seconds with one decimal below ten seconds, whole seconds above. */
|
|
180
|
+
const formatSeconds = (ms: number): string =>
|
|
181
|
+
ms < 10_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms / 1000)}s`;
|
|
182
|
+
|
|
167
183
|
/**
|
|
168
184
|
* Format sync result lines (shared between update and index commands).
|
|
169
185
|
*/
|
|
@@ -194,6 +210,15 @@ export function formatSyncResultLines(
|
|
|
194
210
|
if (c.filesErrored > 0) {
|
|
195
211
|
lines.push(` ${c.filesErrored} errors`);
|
|
196
212
|
}
|
|
213
|
+
// A file stopped at its conversion budget is always named, not only in
|
|
214
|
+
// verbose output: it stays unindexed until a later run finishes it.
|
|
215
|
+
if (!options.verbose) {
|
|
216
|
+
for (const err of c.errors) {
|
|
217
|
+
if (BUDGET_ERROR_CODES.has(err.code)) {
|
|
218
|
+
lines.push(` [${err.code}] ${err.relPath}: ${err.message}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
197
222
|
if (c.filesMarkedInactive > 0) {
|
|
198
223
|
lines.push(` ${c.filesMarkedInactive} marked inactive`);
|
|
199
224
|
}
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
formatMemoryStatusLines,
|
|
28
28
|
} from "../../core/memory-diagnostics";
|
|
29
29
|
import { formatVectorPartitionLines } from "../../core/vector-partition-status";
|
|
30
|
+
import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
|
|
30
31
|
import { ModelCache } from "../../llm/cache";
|
|
31
32
|
import { getActivePreset, resolveModelUri } from "../../llm/registry";
|
|
32
33
|
import { getConnectorVerificationTargets } from "../../serve/connectors";
|
|
@@ -64,9 +65,56 @@ export type StatusResult =
|
|
|
64
65
|
contentTypeBoost: ContentTypeBoostStatus;
|
|
65
66
|
memory: MemoryStatus;
|
|
66
67
|
backgroundIssues: ResidentBackgroundIssue[];
|
|
68
|
+
/** Files whose latest conversion stopped at the per-file budget. */
|
|
69
|
+
budgetStops: BudgetStop[];
|
|
67
70
|
}
|
|
68
71
|
| { success: false; error: string };
|
|
69
72
|
|
|
73
|
+
/** A file left unindexed because its conversion overran its budget. */
|
|
74
|
+
export interface BudgetStop {
|
|
75
|
+
collection: string;
|
|
76
|
+
relPath: string;
|
|
77
|
+
code: string;
|
|
78
|
+
message: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Most recent ingest errors inspected for budget stops. */
|
|
82
|
+
const BUDGET_STOP_SCAN_LIMIT = 100;
|
|
83
|
+
/** Most budget stops listed in status output. */
|
|
84
|
+
const MAX_BUDGET_STOPS = 10;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Files whose latest indexing attempt stopped at the conversion budget, from
|
|
88
|
+
* the recent error log, kept only while the document still carries that code.
|
|
89
|
+
*/
|
|
90
|
+
async function collectBudgetStops(store: SqliteAdapter): Promise<BudgetStop[]> {
|
|
91
|
+
const recent = await store.getRecentErrors(BUDGET_STOP_SCAN_LIMIT);
|
|
92
|
+
if (!recent.ok) return [];
|
|
93
|
+
const stops: BudgetStop[] = [];
|
|
94
|
+
const seen = new Set<string>();
|
|
95
|
+
for (const row of recent.value) {
|
|
96
|
+
const key = `${row.collection}\u0000${row.relPath}`;
|
|
97
|
+
if (seen.has(key)) continue;
|
|
98
|
+
seen.add(key);
|
|
99
|
+
if (!BUDGET_ERROR_CODES.has(row.code)) continue;
|
|
100
|
+
const document = await store.getDocument(row.collection, row.relPath);
|
|
101
|
+
if (
|
|
102
|
+
document.ok &&
|
|
103
|
+
document.value?.active &&
|
|
104
|
+
document.value.lastErrorCode === row.code
|
|
105
|
+
) {
|
|
106
|
+
stops.push({
|
|
107
|
+
collection: row.collection,
|
|
108
|
+
relPath: row.relPath,
|
|
109
|
+
code: row.code,
|
|
110
|
+
message: row.message,
|
|
111
|
+
});
|
|
112
|
+
if (stops.length >= MAX_BUDGET_STOPS) break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return stops;
|
|
116
|
+
}
|
|
117
|
+
|
|
70
118
|
/** A background issue reported by (or about) a running detached resident. */
|
|
71
119
|
export type ResidentBackgroundIssue = BackgroundIssue & {
|
|
72
120
|
process: "serve" | "daemon";
|
|
@@ -368,6 +416,7 @@ export async function status(
|
|
|
368
416
|
contentTypeBoost: buildContentTypeBoostStatus(config.contentTypes ?? []),
|
|
369
417
|
memory: await buildMemoryStatus(store, config.collections),
|
|
370
418
|
backgroundIssues: await collectResidentIssues(),
|
|
419
|
+
budgetStops: await collectBudgetStops(store),
|
|
371
420
|
};
|
|
372
421
|
} finally {
|
|
373
422
|
await store.close();
|
|
@@ -439,6 +488,10 @@ export function formatStatus(
|
|
|
439
488
|
(issue) =>
|
|
440
489
|
` ${issue.process} (pid ${issue.pid}): ${formatBackgroundIssue(issue)}`
|
|
441
490
|
);
|
|
491
|
+
const stopLines = result.budgetStops.map(
|
|
492
|
+
(stop) =>
|
|
493
|
+
` ${stop.collection}/${stop.relPath} [${stop.code}] ${stop.message}`
|
|
494
|
+
);
|
|
442
495
|
return [
|
|
443
496
|
formatTerminal(
|
|
444
497
|
result.status,
|
|
@@ -446,6 +499,13 @@ export function formatStatus(
|
|
|
446
499
|
result.contentTypeBoost,
|
|
447
500
|
result.memory
|
|
448
501
|
),
|
|
502
|
+
...(stopLines.length
|
|
503
|
+
? [
|
|
504
|
+
"",
|
|
505
|
+
"Stopped at conversion budget (retried by the next update):",
|
|
506
|
+
...stopLines,
|
|
507
|
+
]
|
|
508
|
+
: []),
|
|
449
509
|
...(issueLines.length ? ["", "Background issues:", ...issueLines] : []),
|
|
450
510
|
].join("\n");
|
|
451
511
|
}
|
|
@@ -15,7 +15,11 @@ import {
|
|
|
15
15
|
type SyncResult,
|
|
16
16
|
withContentTypeRules,
|
|
17
17
|
} from "../../ingestion";
|
|
18
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
formatSyncResultLines,
|
|
20
|
+
initStore,
|
|
21
|
+
reportSlowConversion,
|
|
22
|
+
} from "./shared";
|
|
19
23
|
|
|
20
24
|
/**
|
|
21
25
|
* Options for update command.
|
|
@@ -66,6 +70,7 @@ export async function update(
|
|
|
66
70
|
{
|
|
67
71
|
gitPull: options.gitPull,
|
|
68
72
|
runUpdateCmd: true,
|
|
73
|
+
onSlowConversion: reportSlowConversion,
|
|
69
74
|
},
|
|
70
75
|
config
|
|
71
76
|
)
|