@gmickel/gno 2.8.2 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.2.zip → gno-browser-clipper-v2.8.4.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.4.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +9 -1
- package/spec/cli.md +77 -6
- package/spec/mcp.md +5 -6
- package/spec/output-schemas/doctor.schema.json +10 -4
- package/spec/output-schemas/status.schema.json +4 -2
- package/src/cli/commands/daemon.ts +7 -2
- package/src/cli/commands/doctor-activation.ts +38 -11
- package/src/cli/commands/doctor.ts +46 -25
- package/src/cli/commands/index-cmd.ts +10 -2
- package/src/cli/commands/links.ts +5 -22
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/status.ts +76 -4
- package/src/cli/commands/update.ts +6 -1
- package/src/cli/commands/vec.ts +42 -2
- package/src/cli/program.ts +2 -2
- package/src/config/types.ts +19 -0
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
- package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
- package/src/converters/adapters/xlsx/adapter.ts +132 -0
- package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
- package/src/converters/registry.ts +62 -6
- package/src/converters/versions.ts +2 -0
- package/src/core/activation-empty.ts +16 -0
- package/src/core/activation-status.ts +23 -5
- package/src/core/audit-freshness.ts +1 -1
- package/src/core/audit-outside-index.ts +48 -3
- package/src/core/change-diff.ts +16 -4
- package/src/core/link-destination-parse.ts +21 -4
- package/src/core/link-inventory-markdown.ts +23 -3
- package/src/core/links.ts +127 -31
- package/src/core/network-boundary-inventory.ts +18 -0
- package/src/index.ts +10 -0
- package/src/ingestion/document-metadata.ts +256 -0
- package/src/ingestion/file-child-env.ts +9 -0
- package/src/ingestion/file-child.ts +106 -0
- package/src/ingestion/file-processor.ts +546 -0
- package/src/ingestion/frontmatter.ts +6 -2
- package/src/ingestion/index.ts +1 -0
- package/src/ingestion/prepare-convert.ts +32 -0
- package/src/ingestion/prepare-file.ts +272 -0
- package/src/ingestion/prepare-worker.ts +43 -0
- package/src/ingestion/record-container.ts +7 -2
- package/src/ingestion/strip.ts +136 -31
- package/src/ingestion/sync-options.ts +7 -1
- package/src/ingestion/sync.ts +207 -400
- package/src/ingestion/types.ts +18 -0
- package/src/llm/native-worker/embedding-identity.ts +15 -3
- package/src/mcp/http-transport.ts +11 -0
- package/src/mcp/tools/links.ts +10 -38
- package/src/serve/activation-health.ts +25 -7
- package/src/serve/public/components/BootstrapStatus.tsx +9 -4
- package/src/serve/resident-request.ts +19 -0
- package/src/serve/resident-runtime.ts +84 -8
- package/src/serve/routes/links.ts +6 -24
- package/src/serve/server.ts +3 -2
- package/src/serve/status.ts +27 -8
- package/src/sessions/config-refresh.ts +3 -2
- package/src/store/sqlite/graph-similarity.ts +5 -13
- package/src/store/vector/partition-index.ts +100 -0
- package/src/store/vector/stored-vectors.ts +20 -15
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.2.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
|
|
|
139
139
|
|
|
140
140
|
<!-- public-truth:current-version -->
|
|
141
141
|
|
|
142
|
-
> Current source version: **v2.8.
|
|
142
|
+
> Current source version: **v2.8.4**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
143
143
|
|
|
144
144
|
<!-- /public-truth -->
|
|
145
145
|
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
956ba0ce42d3b72635b61ca710d7adc8599fbdf08f3e32000d979158cac432c6 gno-browser-clipper-v2.8.4.zip
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gmickel/gno",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.4",
|
|
4
4
|
"description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"embeddings",
|
|
@@ -197,7 +197,15 @@
|
|
|
197
197
|
"mammoth": "1.12.2",
|
|
198
198
|
"mdast-util-from-markdown": "2.0.3",
|
|
199
199
|
"mdast-util-gfm": "3.1.0",
|
|
200
|
+
"mdast-util-gfm-autolink-literal": "2.0.1",
|
|
201
|
+
"mdast-util-gfm-footnote": "2.1.0",
|
|
202
|
+
"mdast-util-gfm-strikethrough": "2.0.0",
|
|
203
|
+
"mdast-util-gfm-task-list-item": "2.0.0",
|
|
200
204
|
"micromark-extension-gfm": "3.0.0",
|
|
205
|
+
"micromark-extension-gfm-autolink-literal": "2.1.0",
|
|
206
|
+
"micromark-extension-gfm-footnote": "2.1.0",
|
|
207
|
+
"micromark-extension-gfm-strikethrough": "2.1.0",
|
|
208
|
+
"micromark-extension-gfm-task-list-item": "2.1.0",
|
|
201
209
|
"mime-types": "2.1.35",
|
|
202
210
|
"minimatch": "10.2.6",
|
|
203
211
|
"nanoid": "6.0.1",
|
package/spec/cli.md
CHANGED
|
@@ -351,7 +351,12 @@ gno status [--json|--md]
|
|
|
351
351
|
```
|
|
352
352
|
|
|
353
353
|
`activation.usable` means at least one configured collection passed its local
|
|
354
|
-
lexical proof. `activation.healthy` means
|
|
354
|
+
lexical proof. `activation.healthy` means at least one collection is configured
|
|
355
|
+
and none failed its proof. A collection with no documents yet (index stage
|
|
356
|
+
`no_documents`, remediation `gno update`) is informational, not a failure: it
|
|
357
|
+
leaves `healthy` true, projects no connector proofs, and terminal output
|
|
358
|
+
lists it as `no documents yet` (`Lexical activation: NO DOCUMENTS YET` when no
|
|
359
|
+
collection has documents).
|
|
355
360
|
Semantic and connector stages remain independent; passive status never starts a
|
|
356
361
|
model runtime or connector process. `gno status` still exits 0 when activation
|
|
357
362
|
is unhealthy so scripts can inspect the structured state.
|
|
@@ -987,7 +992,10 @@ resolves to a regular file inside the workspace, at most 200,000 files), no
|
|
|
987
992
|
file is opened, and no graph edge is created. Wiki targets match files by the workspace
|
|
988
993
|
link rules; a non-Markdown target needs its extension, `.md` is optional. A
|
|
989
994
|
Markdown link matches only the file at its resolved path (NFC,
|
|
990
|
-
case-insensitive)
|
|
995
|
+
case-insensitive); as for wiki links, `.md` is optional, so `[x](Note)` matches
|
|
996
|
+
an unindexed `Note.md`. An indexed `Note.md` (compared under the same NFC,
|
|
997
|
+
Unicode-lowercase key) is not outside the index, and Markdown link resolution
|
|
998
|
+
itself stays exact-path, so that link remains in `links.local-targets`. A
|
|
991
999
|
target missing from the listing stays unresolved; an unreadable folder or the
|
|
992
1000
|
file bound makes the listing incomplete, which the `links.outside-index` rule
|
|
993
1001
|
message states once.
|
|
@@ -995,7 +1003,12 @@ message states once.
|
|
|
995
1003
|
The wiki link parser reads Obsidian's table-escaped alias `[[Note\|Alias]]`
|
|
996
1004
|
as target `Note` with alias `Alias`. Markdown link text may contain balanced
|
|
997
1005
|
square brackets (`[see [1]](note.md)`); a destination containing a square
|
|
998
|
-
bracket is not a link.
|
|
1006
|
+
bracket is not a link. Markdown links follow CommonMark, as Obsidian renders
|
|
1007
|
+
them: a `[` escaped with a backslash (`\[clause](see section 4)`) opens no
|
|
1008
|
+
link, and an inline destination with an unescaped space is plain text unless
|
|
1009
|
+
it is wrapped in angle brackets. `[x](<my note.md>)` and `[x](my%20note.md)`
|
|
1010
|
+
both link to `my note.md`; a link title (`[x](note.md "Title")`) is not part
|
|
1011
|
+
of the target.
|
|
999
1012
|
|
|
1000
1013
|
The JSON contract is versioned as `gno://schemas/audit-report@1.0`. Finding IDs
|
|
1001
1014
|
are stable SHA-256 identities derived from rule, normalized subject/location,
|
|
@@ -1472,6 +1485,33 @@ individual virtual document. The closed `record.adapter` identity contains the
|
|
|
1472
1485
|
adapter ID, version, and configuration fingerprint. `get` and `multi-get`
|
|
1473
1486
|
preserve the same fields.
|
|
1474
1487
|
|
|
1488
|
+
Every file is indexed under the per-file budget `conversion.timeoutMs`
|
|
1489
|
+
(default 60000) and `conversion.maxMemoryMb` (default half of physical memory,
|
|
1490
|
+
at least 2048; process resident memory). Conversion and every post-conversion
|
|
1491
|
+
phase (metadata, code regions, change-journal structure, chunking, link
|
|
1492
|
+
extraction) run in one worker per process, one file at a time; the worker is
|
|
1493
|
+
terminated at either limit, mid-phase, and replaced for the next file.
|
|
1494
|
+
Database reads and writes stay on the main thread. A standalone compiled
|
|
1495
|
+
executable, which cannot start the worker, runs the phases in a child process
|
|
1496
|
+
of itself (internal environment flag) under the same limits; the child's
|
|
1497
|
+
resident memory counts toward `conversion.maxMemoryMb` (Linux `/proc`, `ps`
|
|
1498
|
+
elsewhere; time only on Windows). A processor that cannot start fails the file
|
|
1499
|
+
with `ISOLATION_UNAVAILABLE`; nothing is prepared without the budget. The
|
|
1500
|
+
child is SIGKILLed on process exit, SIGTERM, SIGINT (via the CLI's exit
|
|
1501
|
+
path) and resident shutdown; on Linux it also carries a parent-death signal,
|
|
1502
|
+
so it dies with a SIGKILLed parent, and elsewhere it exits at its next step
|
|
1503
|
+
once its parent is gone.
|
|
1504
|
+
A file over either limit is recorded with error code `TIMEOUT` or
|
|
1505
|
+
`MEMORY_LIMIT` (the message names the running phase), stays unindexed, and is
|
|
1506
|
+
retried by the next sync; the rest of the run continues. `update` and `index` always print such a
|
|
1507
|
+
file in terminal output as `[CODE] <relPath>: <message>` (other per-file
|
|
1508
|
+
errors stay `--verbose` only), and write
|
|
1509
|
+
`Still converting <collection>/<relPath> after <n>s (budget <m>s)` to stderr
|
|
1510
|
+
once a file has been processing for 10 seconds or half its budget, whichever
|
|
1511
|
+
is sooner.
|
|
1512
|
+
`gno status` lists stopped files under "Stopped at conversion budget" in
|
|
1513
|
+
terminal output (JSON output is unchanged).
|
|
1514
|
+
|
|
1475
1515
|
File/export adapter iteration has a 60-second deadline in addition to the byte,
|
|
1476
1516
|
record, metadata, and total-character caps. A partial import that retains valid
|
|
1477
1517
|
siblings exits successfully but is never silent: terminal output includes the
|
|
@@ -2976,14 +3016,22 @@ misconfigured `findings` block (see [Daemon Mode](../docs/DAEMON.md#scheduled-fi
|
|
|
2976
3016
|
The `embedding-fingerprint` check is additive doctor-only diagnostics. It uses
|
|
2977
3017
|
the active embed model and stored vector dimensions to report the current
|
|
2978
3018
|
freshness fingerprint, pending/stale chunks, legacy empty-fingerprint vectors,
|
|
2979
|
-
and stored fingerprint groups.
|
|
3019
|
+
and stored fingerprint groups. On an index with an activated vector partition
|
|
3020
|
+
for the embed model (every index `gno embed` wrote since 2.7), the groups are
|
|
3021
|
+
that partition with its current chunk count and there are no legacy vectors;
|
|
3022
|
+
legacy `content_vectors` groups are read only before any partition activates.
|
|
3023
|
+
Stale, legacy, and mixed groups are warnings;
|
|
2980
3024
|
recover with `gno embed`, or `gno embed --force` if vectors still look stale.
|
|
2981
3025
|
|
|
2982
3026
|
The additive `activation` object uses the same contract as `gno status` and
|
|
2983
3027
|
`GET /api/status`. Doctor performs only the local lexical proof. It never starts
|
|
2984
3028
|
connector children or initializes/downloads models. A failed lexical proof adds
|
|
2985
3029
|
the `retrieval-activation` error check and exits 2 after writing the complete
|
|
2986
|
-
result; no duplicate error is written to stderr.
|
|
3030
|
+
result; no duplicate error is written to stderr. A collection with no
|
|
3031
|
+
documents yet is not a failed proof: when it is the only activation problem,
|
|
3032
|
+
`retrieval-activation` has status `info` (terminal icon `i`) and lists the
|
|
3033
|
+
collection with its remediation (add files, then `gno update`). Check status
|
|
3034
|
+
is one of `ok`, `info`, `warn` or `error`; only `error` fails doctor. Connector failure or projection
|
|
2987
3035
|
truncation adds a warning and makes the structured doctor result non-healthy,
|
|
2988
3036
|
but preserves exit 0 when lexical proof and all other required checks pass. An
|
|
2989
3037
|
omitted target/collection pair has no inferred result.
|
|
@@ -3988,7 +4036,10 @@ gno similar <doc> [-n, --limit <num>] [--threshold <num>] [--cross-collection] [
|
|
|
3988
4036
|
|
|
3989
4037
|
- Finds documents semantically similar to the source document
|
|
3990
4038
|
- Requires embeddings to be generated (`gno embed`)
|
|
3991
|
-
-
|
|
4039
|
+
- Source vector: the stored vector of the document's first chunk (the
|
|
4040
|
+
lowest-seq chunk whose vector is current), read from the active vector
|
|
4041
|
+
partition. `gno_similar` (MCP), `GET /api/doc/:id/similar` (REST) and graph
|
|
4042
|
+
similarity edges use the same rule, so scores match across surfaces
|
|
3992
4043
|
- By default, limits results to same collection
|
|
3993
4044
|
|
|
3994
4045
|
**Output (JSON):**
|
|
@@ -4203,6 +4254,9 @@ gno changes --follow --jsonl [--cursor <cursor>] [--collection <name>]
|
|
|
4203
4254
|
old/new identity and hash snapshots, normalized structural deltas, pagination,
|
|
4204
4255
|
cursor-expiry, and retention-truncation disclosure.
|
|
4205
4256
|
- The journal never returns source bodies.
|
|
4257
|
+
- Link additions and removals in `structureDelta.links` are recorded only for
|
|
4258
|
+
Markdown sources, matching link extraction; content changes of other
|
|
4259
|
+
documents (code, text, converted files, records) are still journaled.
|
|
4206
4260
|
|
|
4207
4261
|
**Follow mode (`--follow --jsonl`)** streams journal events as they land and is
|
|
4208
4262
|
the durable automation input for consumers that resume across restarts.
|
|
@@ -4444,6 +4498,21 @@ gno serve --detach --stop
|
|
|
4444
4498
|
|
|
4445
4499
|
---
|
|
4446
4500
|
|
|
4501
|
+
#### Config reload while running
|
|
4502
|
+
|
|
4503
|
+
A running `gno serve` or `gno daemon` re-reads its config file before each
|
|
4504
|
+
request (REST, Web UI, HTTP MCP, daemon status) and every 2 seconds, and adopts
|
|
4505
|
+
changes to `collections` and `contexts` made by the CLI (`gno collection
|
|
4506
|
+
add/remove`) or by hand: status, MCP `gno_status`, the watcher and search scope
|
|
4507
|
+
follow without a restart. A newly added collection is watched from then on; its
|
|
4508
|
+
existing files are indexed by `gno update` (or the next start's initial sync).
|
|
4509
|
+
An adoption that changes the collection set or policies moves the egress policy
|
|
4510
|
+
epoch as a REST collection change does. The checks are the sessions config
|
|
4511
|
+
refresh's: an unreadable config file answers an error (`500`, sessions code
|
|
4512
|
+
`SESSIONS_RUNTIME_FAILURE`), never the stale config, and a config bound to
|
|
4513
|
+
another index is refused. Other settings (models, gateway, sessions sources
|
|
4514
|
+
outside the sessions routes) keep their restart behaviour.
|
|
4515
|
+
|
|
4447
4516
|
### gno daemon
|
|
4448
4517
|
|
|
4449
4518
|
Start a headless long-running watcher process for continuous indexing.
|
|
@@ -4483,6 +4552,8 @@ is blocked.
|
|
|
4483
4552
|
|
|
4484
4553
|
- Opens DB once at startup
|
|
4485
4554
|
- Loads config and requires at least one configured collection
|
|
4555
|
+
- Follows collection edits to the config file while running (see
|
|
4556
|
+
[Config reload](#config-reload-while-running))
|
|
4486
4557
|
- Starts the same watcher + embed scheduler used by `gno serve`
|
|
4487
4558
|
- Exact contained eligible paths always use targeted content-hash sync.
|
|
4488
4559
|
Ambiguous temp/directory/missing-name/recursive-delete events use bounded,
|
package/spec/mcp.md
CHANGED
|
@@ -2188,12 +2188,11 @@ Find semantically similar documents using vector embeddings.
|
|
|
2188
2188
|
|
|
2189
2189
|
**Algorithm:**
|
|
2190
2190
|
|
|
2191
|
-
1.
|
|
2192
|
-
2.
|
|
2193
|
-
3.
|
|
2194
|
-
4.
|
|
2195
|
-
5.
|
|
2196
|
-
6. Return top N similar documents with scores
|
|
2191
|
+
1. Take the stored vector of the source document's first chunk (the lowest-seq chunk whose vector is current) from the active vector partition (legacy `content_vectors` only before any partition activates); no model is loaded. This is the same source vector `gno similar`, `GET /api/doc/:id/similar` and graph similarity edges use, so scores match across surfaces
|
|
2192
|
+
2. Normalize it to unit length
|
|
2193
|
+
3. Search for nearest neighbors using sqlite-vec
|
|
2194
|
+
4. Exclude self and filter by collection if not crossCollection
|
|
2195
|
+
5. Return top N similar documents with scores
|
|
2197
2196
|
|
|
2198
2197
|
**Errors:**
|
|
2199
2198
|
|
|
@@ -24,8 +24,14 @@
|
|
|
24
24
|
],
|
|
25
25
|
"properties": {
|
|
26
26
|
"schemaVersion": { "const": "1.0" },
|
|
27
|
-
"usable": {
|
|
28
|
-
|
|
27
|
+
"usable": {
|
|
28
|
+
"type": "boolean",
|
|
29
|
+
"description": "At least one collection passed the lexical retrieval proof"
|
|
30
|
+
},
|
|
31
|
+
"healthy": {
|
|
32
|
+
"type": "boolean",
|
|
33
|
+
"description": "At least one collection is configured and none failed its lexical retrieval proof. A collection with no documents yet (remediation code no_documents, command gno update) is informational, not a failure."
|
|
34
|
+
},
|
|
29
35
|
"collections": {
|
|
30
36
|
"type": "array",
|
|
31
37
|
"items": {
|
|
@@ -229,8 +235,8 @@
|
|
|
229
235
|
},
|
|
230
236
|
"status": {
|
|
231
237
|
"type": "string",
|
|
232
|
-
"enum": ["ok", "warn", "error"],
|
|
233
|
-
"description": "Check result status"
|
|
238
|
+
"enum": ["ok", "info", "warn", "error"],
|
|
239
|
+
"description": "Check result status. info reports a state that needs no repair, such as a collection with no documents yet"
|
|
234
240
|
},
|
|
235
241
|
"message": {
|
|
236
242
|
"type": "string",
|
|
@@ -347,10 +347,12 @@
|
|
|
347
347
|
"const": "1.0"
|
|
348
348
|
},
|
|
349
349
|
"usable": {
|
|
350
|
-
"type": "boolean"
|
|
350
|
+
"type": "boolean",
|
|
351
|
+
"description": "At least one collection passed the lexical retrieval proof"
|
|
351
352
|
},
|
|
352
353
|
"healthy": {
|
|
353
|
-
"type": "boolean"
|
|
354
|
+
"type": "boolean",
|
|
355
|
+
"description": "At least one collection is configured and none failed its lexical retrieval proof. A collection with no documents yet (remediation code no_documents, command gno update) is informational, not a failure."
|
|
354
356
|
},
|
|
355
357
|
"collections": {
|
|
356
358
|
"type": "array",
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
resolveHttpGatewayConfig,
|
|
17
17
|
} from "../../mcp/http-security";
|
|
18
18
|
import { startBackgroundRuntime } from "../../serve/background-runtime";
|
|
19
|
+
import { refreshResidentConfig } from "../../serve/resident-request";
|
|
19
20
|
import { handleResidentStatus, handleStatus } from "../../serve/routes/api";
|
|
20
21
|
import { createMcpHttpGateway } from "../../serve/routes/mcp";
|
|
21
22
|
import { SessionsError } from "../../sessions/types";
|
|
@@ -106,6 +107,8 @@ export const authorizeDaemonStatus = async (
|
|
|
106
107
|
): Promise<Response> => {
|
|
107
108
|
const authorization = await gateway.security.authorize(request, server);
|
|
108
109
|
if (!authorization.ok) return authorization.response;
|
|
110
|
+
const refreshFailure = await refreshResidentConfig(runtime);
|
|
111
|
+
if (refreshFailure) return refreshFailure;
|
|
109
112
|
try {
|
|
110
113
|
enforceCollectionEgress({
|
|
111
114
|
collections: runtime.config.collections,
|
|
@@ -132,13 +135,15 @@ export const authorizeDaemonStatus = async (
|
|
|
132
135
|
}
|
|
133
136
|
};
|
|
134
137
|
|
|
135
|
-
export function handleDaemonAppStatus(
|
|
138
|
+
export async function handleDaemonAppStatus(
|
|
136
139
|
runtime: ResidentRuntime,
|
|
137
140
|
host: string
|
|
138
|
-
): Promise<Response>
|
|
141
|
+
): Promise<Response> {
|
|
139
142
|
if (!isHttpGatewayLoopbackBind(host)) {
|
|
140
143
|
return new Response(null, { status: 404 });
|
|
141
144
|
}
|
|
145
|
+
const refreshFailure = await refreshResidentConfig(runtime);
|
|
146
|
+
if (refreshFailure) return refreshFailure;
|
|
142
147
|
return handleStatus(runtime.ctxHolder.current, {
|
|
143
148
|
getResidentStatus: () => runtime.getStatus(),
|
|
144
149
|
});
|
|
@@ -6,6 +6,7 @@ import type { StorePort } from "../../store/types";
|
|
|
6
6
|
import type { DoctorCheck } from "./doctor";
|
|
7
7
|
|
|
8
8
|
import { getIndexDbPath, getModelsCachePath } from "../../app/constants";
|
|
9
|
+
import { isEmptyActivationCollection } from "../../core/activation-empty";
|
|
9
10
|
import { buildActivationStatus } from "../../core/activation-status";
|
|
10
11
|
import { ModelCache } from "../../llm/cache";
|
|
11
12
|
import { getActivePreset } from "../../llm/registry";
|
|
@@ -75,28 +76,54 @@ export async function buildDoctorActivation(
|
|
|
75
76
|
}
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
const plural = (count: number, noun: string): string =>
|
|
80
|
+
`${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
81
|
+
|
|
78
82
|
export function checkRetrievalActivation(
|
|
79
83
|
activation: ActivationStatus
|
|
80
84
|
): DoctorCheck {
|
|
85
|
+
const ready = activation.collections.filter((collection) => collection.ready);
|
|
86
|
+
const empty = activation.collections.filter(isEmptyActivationCollection);
|
|
87
|
+
const emptyDetails = empty.flatMap(({ collection, remediation }) => [
|
|
88
|
+
`${collection}: no documents indexed yet (informational)`,
|
|
89
|
+
`Add files, then run: ${remediation?.command ?? "gno update"}`,
|
|
90
|
+
]);
|
|
91
|
+
|
|
81
92
|
if (activation.healthy) {
|
|
82
93
|
const semanticStates = [
|
|
83
94
|
...new Set(
|
|
84
|
-
|
|
85
|
-
({ semanticAvailability }) => semanticAvailability.code
|
|
86
|
-
)
|
|
95
|
+
ready.map(({ semanticAvailability }) => semanticAvailability.code)
|
|
87
96
|
),
|
|
88
97
|
];
|
|
98
|
+
const semanticDetail =
|
|
99
|
+
ready.length > 0
|
|
100
|
+
? [
|
|
101
|
+
`Semantic retrieval remains separate (${semanticStates.join(", ")}).`,
|
|
102
|
+
]
|
|
103
|
+
: [];
|
|
104
|
+
if (empty.length === 0) {
|
|
105
|
+
return {
|
|
106
|
+
name: "retrieval-activation",
|
|
107
|
+
status: "ok",
|
|
108
|
+
message: `${plural(ready.length, "collection")} passed lexical retrieval proof`,
|
|
109
|
+
details: semanticDetail,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
89
112
|
return {
|
|
90
113
|
name: "retrieval-activation",
|
|
91
|
-
status: "
|
|
92
|
-
message:
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
114
|
+
status: "info",
|
|
115
|
+
message:
|
|
116
|
+
ready.length > 0
|
|
117
|
+
? `${plural(ready.length, "collection")} passed lexical retrieval proof; ${plural(empty.length, "collection")} with no documents yet`
|
|
118
|
+
: `No documents indexed yet in ${plural(empty.length, "collection")}`,
|
|
119
|
+
details: [...emptyDetails, ...semanticDetail],
|
|
96
120
|
};
|
|
97
121
|
}
|
|
98
122
|
|
|
99
|
-
const failed = activation.collections.filter(
|
|
123
|
+
const failed = activation.collections.filter(
|
|
124
|
+
(collection) =>
|
|
125
|
+
!collection.ready && !isEmptyActivationCollection(collection)
|
|
126
|
+
);
|
|
100
127
|
const details = failed.flatMap(({ collection, remediation }) =>
|
|
101
128
|
remediation
|
|
102
129
|
? [
|
|
@@ -112,9 +139,9 @@ export function checkRetrievalActivation(
|
|
|
112
139
|
activation.collections.length === 0
|
|
113
140
|
? "No collections configured. Run: gno collection add"
|
|
114
141
|
: activation.usable
|
|
115
|
-
? `${failed.length
|
|
142
|
+
? `${plural(failed.length, "collection")} failed lexical retrieval proof`
|
|
116
143
|
: "No configured collection passed lexical retrieval proof",
|
|
117
|
-
details,
|
|
144
|
+
details: [...details, ...emptyDetails],
|
|
118
145
|
};
|
|
119
146
|
}
|
|
120
147
|
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
resolveFindingsSchedule,
|
|
32
32
|
} from "../../core/findings-run-state";
|
|
33
33
|
import { formatVectorPartitionLines } from "../../core/vector-partition-status";
|
|
34
|
+
import { getEmbeddingFingerprint } from "../../embed/fingerprint";
|
|
34
35
|
import { getCodeChunkingStatus } from "../../ingestion/chunker";
|
|
35
36
|
import { ModelCache } from "../../llm/cache";
|
|
36
37
|
import { getActivePreset, resolveModelUri } from "../../llm/registry";
|
|
@@ -42,6 +43,8 @@ import {
|
|
|
42
43
|
getLoadAttempts,
|
|
43
44
|
} from "../../store/sqlite/setup";
|
|
44
45
|
import { getStoredEmbeddingFingerprint } from "../../store/vector/freshness";
|
|
46
|
+
import { currentOwnerCount } from "../../store/vector/runtime-compat";
|
|
47
|
+
import { storedVectorPartition } from "../../store/vector/status";
|
|
45
48
|
import {
|
|
46
49
|
buildDoctorActivation,
|
|
47
50
|
checkConnectorActivation,
|
|
@@ -52,7 +55,8 @@ import {
|
|
|
52
55
|
// Types
|
|
53
56
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
54
57
|
|
|
55
|
-
|
|
58
|
+
/** `info` reports a state that needs no repair, such as an empty collection. */
|
|
59
|
+
export type DoctorCheckStatus = "ok" | "info" | "warn" | "error";
|
|
56
60
|
|
|
57
61
|
export interface DoctorCheck {
|
|
58
62
|
name: string;
|
|
@@ -319,10 +323,14 @@ async function checkEmbeddingFingerprints(
|
|
|
319
323
|
];
|
|
320
324
|
}
|
|
321
325
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
+
// An index embedded since 2.7 keeps its vectors in the active partition;
|
|
327
|
+
// legacy `content_vectors` count only before any partition activates.
|
|
328
|
+
const partition = storedVectorPartition(db, model);
|
|
329
|
+
const legacyChunks = partition
|
|
330
|
+
? 0
|
|
331
|
+
: (db
|
|
332
|
+
.query<{ count: number }, [string]>(
|
|
333
|
+
`
|
|
326
334
|
SELECT COUNT(*) as count
|
|
327
335
|
FROM content_vectors v
|
|
328
336
|
JOIN content_chunks c
|
|
@@ -336,12 +344,23 @@ async function checkEmbeddingFingerprints(
|
|
|
336
344
|
AND d.active = 1
|
|
337
345
|
)
|
|
338
346
|
`
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const groups =
|
|
343
|
-
|
|
344
|
-
|
|
347
|
+
)
|
|
348
|
+
.get(model)?.count ?? 0);
|
|
349
|
+
|
|
350
|
+
const groups = partition
|
|
351
|
+
? [
|
|
352
|
+
{
|
|
353
|
+
model,
|
|
354
|
+
fingerprint: getEmbeddingFingerprint({
|
|
355
|
+
modelUri: model,
|
|
356
|
+
dimensions: partition.dimensions,
|
|
357
|
+
}),
|
|
358
|
+
count: currentOwnerCount(db, partition.partitionId),
|
|
359
|
+
},
|
|
360
|
+
]
|
|
361
|
+
: db
|
|
362
|
+
.query<{ model: string; fingerprint: string; count: number }, []>(
|
|
363
|
+
`
|
|
345
364
|
SELECT
|
|
346
365
|
v.model as model,
|
|
347
366
|
v.embed_fingerprint as fingerprint,
|
|
@@ -358,18 +377,18 @@ async function checkEmbeddingFingerprints(
|
|
|
358
377
|
GROUP BY v.model, v.embed_fingerprint
|
|
359
378
|
ORDER BY count DESC, v.model ASC, v.embed_fingerprint ASC
|
|
360
379
|
`
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
const mixedGroups =
|
|
380
|
+
)
|
|
381
|
+
.all();
|
|
382
|
+
const fingerprintGroups = groups.map((group) => ({
|
|
383
|
+
model: group.model,
|
|
384
|
+
fingerprint: group.fingerprint,
|
|
385
|
+
count: group.count,
|
|
386
|
+
current:
|
|
387
|
+
group.model === model && group.fingerprint === currentFingerprint,
|
|
388
|
+
legacy: group.fingerprint === "",
|
|
389
|
+
}));
|
|
390
|
+
|
|
391
|
+
const mixedGroups = fingerprintGroups.length;
|
|
373
392
|
const pendingChunks = statusResult.value.embeddingBacklog;
|
|
374
393
|
const hasWarnings =
|
|
375
394
|
pendingChunks > 0 || legacyChunks > 0 || mixedGroups > 1;
|
|
@@ -380,7 +399,7 @@ async function checkEmbeddingFingerprints(
|
|
|
380
399
|
pendingChunks,
|
|
381
400
|
legacyChunks,
|
|
382
401
|
mixedGroups,
|
|
383
|
-
groups,
|
|
402
|
+
groups: fingerprintGroups,
|
|
384
403
|
};
|
|
385
404
|
|
|
386
405
|
const message =
|
|
@@ -392,7 +411,7 @@ async function checkEmbeddingFingerprints(
|
|
|
392
411
|
if (hasWarnings) {
|
|
393
412
|
details.push("Run: gno embed");
|
|
394
413
|
details.push("If vectors still look stale, run: gno embed --force");
|
|
395
|
-
for (const group of
|
|
414
|
+
for (const group of fingerprintGroups) {
|
|
396
415
|
const label = describeFingerprintGroup(group);
|
|
397
416
|
const fingerprint = group.legacy
|
|
398
417
|
? "(empty)"
|
|
@@ -736,6 +755,8 @@ function statusIcon(status: DoctorCheckStatus): string {
|
|
|
736
755
|
switch (status) {
|
|
737
756
|
case "ok":
|
|
738
757
|
return "✓";
|
|
758
|
+
case "info":
|
|
759
|
+
return "i";
|
|
739
760
|
case "warn":
|
|
740
761
|
return "!";
|
|
741
762
|
case "error":
|
|
@@ -28,7 +28,11 @@ import {
|
|
|
28
28
|
withContentTypeRules,
|
|
29
29
|
} from "../../ingestion";
|
|
30
30
|
import { type EmbedResult, embedStageOutcome } from "./embed";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
formatSyncResultLines,
|
|
33
|
+
initStore,
|
|
34
|
+
reportSlowConversion,
|
|
35
|
+
} from "./shared";
|
|
32
36
|
|
|
33
37
|
/**
|
|
34
38
|
* Options for index command.
|
|
@@ -202,7 +206,11 @@ export async function index(options: IndexOptions = {}): Promise<IndexResult> {
|
|
|
202
206
|
collections,
|
|
203
207
|
store,
|
|
204
208
|
withContentTypeRules(
|
|
205
|
-
{
|
|
209
|
+
{
|
|
210
|
+
gitPull: options.gitPull,
|
|
211
|
+
runUpdateCmd: true,
|
|
212
|
+
onSlowConversion: reportSlowConversion,
|
|
213
|
+
},
|
|
206
214
|
config
|
|
207
215
|
)
|
|
208
216
|
);
|
|
@@ -587,21 +587,17 @@ export async function similar(
|
|
|
587
587
|
}
|
|
588
588
|
const db = store.getRawDb();
|
|
589
589
|
|
|
590
|
-
//
|
|
590
|
+
// Similarity source vector (first chunk, unit length), active partition
|
|
591
591
|
const {
|
|
592
|
-
|
|
592
|
+
readSimilaritySourceVectors,
|
|
593
593
|
resolveStoredVectorSource,
|
|
594
594
|
similarityHitDocuments,
|
|
595
595
|
storedVectorSearchOptions,
|
|
596
596
|
} = await import("../../store/vector/stored-vectors.js");
|
|
597
597
|
const source = resolveStoredVectorSource(db, modelPreset.embed);
|
|
598
|
-
const
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
source,
|
|
602
|
-
[{ id: doc.id, mirrorHash: doc.mirrorHash }],
|
|
603
|
-
{ firstChunkOnly: true }
|
|
604
|
-
).get(doc.id) ?? [];
|
|
598
|
+
const embedding = readSimilaritySourceVectors(db, source, [
|
|
599
|
+
{ id: doc.id, mirrorHash: doc.mirrorHash },
|
|
600
|
+
]).get(doc.id);
|
|
605
601
|
|
|
606
602
|
if (!embedding) {
|
|
607
603
|
return {
|
|
@@ -610,20 +606,7 @@ export async function similar(
|
|
|
610
606
|
isValidation: true,
|
|
611
607
|
};
|
|
612
608
|
}
|
|
613
|
-
|
|
614
|
-
// Normalize embedding for cosine similarity
|
|
615
609
|
const dimensions = embedding.length;
|
|
616
|
-
let norm = 0;
|
|
617
|
-
for (let i = 0; i < dimensions; i++) {
|
|
618
|
-
const val = embedding[i] ?? 0;
|
|
619
|
-
norm += val * val;
|
|
620
|
-
}
|
|
621
|
-
norm = Math.sqrt(norm);
|
|
622
|
-
if (norm > 0) {
|
|
623
|
-
for (let i = 0; i < dimensions; i++) {
|
|
624
|
-
embedding[i] = (embedding[i] ?? 0) / norm;
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
610
|
|
|
628
611
|
// Create vector index port for search
|
|
629
612
|
const { createVectorIndexPort } =
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { Collection, Config } from "../../config/types";
|
|
9
|
-
import type { SyncResult } from "../../ingestion";
|
|
9
|
+
import type { SlowConversionEvent, SyncResult } from "../../ingestion";
|
|
10
10
|
import type { SearchResults } from "../../pipeline/types";
|
|
11
11
|
|
|
12
12
|
import {
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
loadConfig,
|
|
21
21
|
writeConfigWarningsToStderr,
|
|
22
22
|
} from "../../config";
|
|
23
|
+
import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
|
|
23
24
|
import { SqliteAdapter } from "../../store/sqlite/adapter";
|
|
24
25
|
import { assertCliSessionBinding } from "../session-binding";
|
|
25
26
|
|
|
@@ -164,6 +165,21 @@ export function decorateSearchResultsForIndex(
|
|
|
164
165
|
};
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Name a file whose conversion is still running past the notice threshold,
|
|
170
|
+
* so an operator can see which file a slow run is working on (stderr, so
|
|
171
|
+
* JSON output on stdout stays parseable).
|
|
172
|
+
*/
|
|
173
|
+
export function reportSlowConversion(event: SlowConversionEvent): void {
|
|
174
|
+
process.stderr.write(
|
|
175
|
+
`Still converting ${event.collection}/${event.relPath} after ${formatSeconds(event.elapsedMs)} (budget ${formatSeconds(event.budgetMs)})\n`
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Seconds with one decimal below ten seconds, whole seconds above. */
|
|
180
|
+
const formatSeconds = (ms: number): string =>
|
|
181
|
+
ms < 10_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms / 1000)}s`;
|
|
182
|
+
|
|
167
183
|
/**
|
|
168
184
|
* Format sync result lines (shared between update and index commands).
|
|
169
185
|
*/
|
|
@@ -194,6 +210,15 @@ export function formatSyncResultLines(
|
|
|
194
210
|
if (c.filesErrored > 0) {
|
|
195
211
|
lines.push(` ${c.filesErrored} errors`);
|
|
196
212
|
}
|
|
213
|
+
// A file stopped at its conversion budget is always named, not only in
|
|
214
|
+
// verbose output: it stays unindexed until a later run finishes it.
|
|
215
|
+
if (!options.verbose) {
|
|
216
|
+
for (const err of c.errors) {
|
|
217
|
+
if (BUDGET_ERROR_CODES.has(err.code)) {
|
|
218
|
+
lines.push(` [${err.code}] ${err.relPath}: ${err.message}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
197
222
|
if (c.filesMarkedInactive > 0) {
|
|
198
223
|
lines.push(` ${c.filesMarkedInactive} marked inactive`);
|
|
199
224
|
}
|