@gmickel/gno 2.8.2 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +1 -1
  2. package/browser-extension/artifacts/{gno-browser-clipper-v2.8.2.zip → gno-browser-clipper-v2.8.3.zip} +0 -0
  3. package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +1 -0
  4. package/browser-extension/dist/manifest.json +1 -1
  5. package/package.json +9 -1
  6. package/spec/cli.md +66 -4
  7. package/spec/mcp.md +5 -6
  8. package/src/cli/commands/daemon.ts +7 -2
  9. package/src/cli/commands/doctor.ts +42 -24
  10. package/src/cli/commands/index-cmd.ts +10 -2
  11. package/src/cli/commands/links.ts +5 -22
  12. package/src/cli/commands/shared.ts +26 -1
  13. package/src/cli/commands/status.ts +60 -0
  14. package/src/cli/commands/update.ts +6 -1
  15. package/src/cli/commands/vec.ts +42 -2
  16. package/src/cli/program.ts +2 -2
  17. package/src/config/types.ts +19 -0
  18. package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
  19. package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
  20. package/src/converters/adapters/xlsx/adapter.ts +132 -0
  21. package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
  22. package/src/converters/registry.ts +62 -6
  23. package/src/converters/versions.ts +2 -0
  24. package/src/core/audit-freshness.ts +1 -1
  25. package/src/core/audit-outside-index.ts +48 -3
  26. package/src/core/change-diff.ts +16 -4
  27. package/src/core/link-destination-parse.ts +21 -4
  28. package/src/core/link-inventory-markdown.ts +23 -3
  29. package/src/core/links.ts +74 -16
  30. package/src/core/network-boundary-inventory.ts +18 -0
  31. package/src/index.ts +10 -0
  32. package/src/ingestion/document-metadata.ts +256 -0
  33. package/src/ingestion/file-child-env.ts +9 -0
  34. package/src/ingestion/file-child.ts +106 -0
  35. package/src/ingestion/file-processor.ts +546 -0
  36. package/src/ingestion/frontmatter.ts +6 -2
  37. package/src/ingestion/index.ts +1 -0
  38. package/src/ingestion/prepare-convert.ts +32 -0
  39. package/src/ingestion/prepare-file.ts +272 -0
  40. package/src/ingestion/prepare-worker.ts +43 -0
  41. package/src/ingestion/record-container.ts +7 -2
  42. package/src/ingestion/strip.ts +71 -8
  43. package/src/ingestion/sync-options.ts +7 -1
  44. package/src/ingestion/sync.ts +207 -400
  45. package/src/ingestion/types.ts +18 -0
  46. package/src/llm/native-worker/embedding-identity.ts +15 -3
  47. package/src/mcp/http-transport.ts +11 -0
  48. package/src/mcp/tools/links.ts +10 -38
  49. package/src/serve/resident-request.ts +19 -0
  50. package/src/serve/resident-runtime.ts +84 -8
  51. package/src/serve/routes/links.ts +6 -24
  52. package/src/serve/server.ts +3 -2
  53. package/src/sessions/config-refresh.ts +3 -2
  54. package/src/store/sqlite/graph-similarity.ts +5 -13
  55. package/src/store/vector/partition-index.ts +100 -0
  56. package/src/store/vector/stored-vectors.ts +20 -15
  57. package/browser-extension/artifacts/gno-browser-clipper-v2.8.2.zip.sha256 +0 -1
package/README.md CHANGED
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
139
139
 
140
140
  <!-- public-truth:current-version -->
141
141
 
142
- > Current source version: **v2.8.2**. See [CHANGELOG.md](./CHANGELOG.md).
142
+ > Current source version: **v2.8.3**. See [CHANGELOG.md](./CHANGELOG.md).
143
143
 
144
144
  <!-- /public-truth -->
145
145
 
@@ -0,0 +1 @@
1
+ 7fe15ce08a24f4dae4ac6694f62bb26d94bbc47d6abc09826f29267ace10558d gno-browser-clipper-v2.8.3.zip
@@ -21,5 +21,5 @@
21
21
  "content_security_policy": {
22
22
  "extension_pages": "script-src 'self'; object-src 'none'; connect-src http://127.0.0.1:*"
23
23
  },
24
- "version": "2.8.2"
24
+ "version": "2.8.3"
25
25
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gmickel/gno",
3
- "version": "2.8.2",
3
+ "version": "2.8.3",
4
4
  "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
5
5
  "keywords": [
6
6
  "embeddings",
@@ -197,7 +197,15 @@
197
197
  "mammoth": "1.12.2",
198
198
  "mdast-util-from-markdown": "2.0.3",
199
199
  "mdast-util-gfm": "3.1.0",
200
+ "mdast-util-gfm-autolink-literal": "2.0.1",
201
+ "mdast-util-gfm-footnote": "2.1.0",
202
+ "mdast-util-gfm-strikethrough": "2.0.0",
203
+ "mdast-util-gfm-task-list-item": "2.0.0",
200
204
  "micromark-extension-gfm": "3.0.0",
205
+ "micromark-extension-gfm-autolink-literal": "2.1.0",
206
+ "micromark-extension-gfm-footnote": "2.1.0",
207
+ "micromark-extension-gfm-strikethrough": "2.1.0",
208
+ "micromark-extension-gfm-task-list-item": "2.1.0",
201
209
  "mime-types": "2.1.35",
202
210
  "minimatch": "10.2.6",
203
211
  "nanoid": "6.0.1",
package/spec/cli.md CHANGED
@@ -987,7 +987,10 @@ resolves to a regular file inside the workspace, at most 200,000 files), no
987
987
  file is opened, and no graph edge is created. Wiki targets match files by the workspace
988
988
  link rules; a non-Markdown target needs its extension, `.md` is optional. A
989
989
  Markdown link matches only the file at its resolved path (NFC,
990
- case-insensitive). A
990
+ case-insensitive); as for wiki links, `.md` is optional, so `[x](Note)` matches
991
+ an unindexed `Note.md`. An indexed `Note.md` (compared under the same NFC,
992
+ Unicode-lowercase key) is not outside the index, and Markdown link resolution
993
+ itself stays exact-path, so that link remains in `links.local-targets`. A
991
994
  target missing from the listing stays unresolved; an unreadable folder or the
992
995
  file bound makes the listing incomplete, which the `links.outside-index` rule
993
996
  message states once.
@@ -995,7 +998,12 @@ message states once.
995
998
  The wiki link parser reads Obsidian's table-escaped alias `[[Note\|Alias]]`
996
999
  as target `Note` with alias `Alias`. Markdown link text may contain balanced
997
1000
  square brackets (`[see [1]](note.md)`); a destination containing a square
998
- bracket is not a link.
1001
+ bracket is not a link. Markdown links follow CommonMark, as Obsidian renders
1002
+ them: a `[` escaped with a backslash (`\[clause](see section 4)`) opens no
1003
+ link, and an inline destination with an unescaped space is plain text unless
1004
+ it is wrapped in angle brackets. `[x](<my note.md>)` and `[x](my%20note.md)`
1005
+ both link to `my note.md`; a link title (`[x](note.md "Title")`) is not part
1006
+ of the target.
999
1007
 
1000
1008
  The JSON contract is versioned as `gno://schemas/audit-report@1.0`. Finding IDs
1001
1009
  are stable SHA-256 identities derived from rule, normalized subject/location,
@@ -1472,6 +1480,33 @@ individual virtual document. The closed `record.adapter` identity contains the
1472
1480
  adapter ID, version, and configuration fingerprint. `get` and `multi-get`
1473
1481
  preserve the same fields.
1474
1482
 
1483
+ Every file is indexed under the per-file budget `conversion.timeoutMs`
1484
+ (default 60000) and `conversion.maxMemoryMb` (default half of physical memory,
1485
+ at least 2048; process resident memory). Conversion and every post-conversion
1486
+ phase (metadata, code regions, change-journal structure, chunking, link
1487
+ extraction) run in one worker per process, one file at a time; the worker is
1488
+ terminated at either limit, mid-phase, and replaced for the next file.
1489
+ Database reads and writes stay on the main thread. A standalone compiled
1490
+ executable, which cannot start the worker, runs the phases in a child process
1491
+ of itself (internal environment flag) under the same limits; the child's
1492
+ resident memory counts toward `conversion.maxMemoryMb` (Linux `/proc`, `ps`
1493
+ elsewhere; time only on Windows). A processor that cannot start fails the file
1494
+ with `ISOLATION_UNAVAILABLE`; nothing is prepared without the budget. The
1495
+ child is SIGKILLed on process exit, SIGTERM, SIGINT (via the CLI's exit
1496
+ path) and resident shutdown; on Linux it also carries a parent-death signal,
1497
+ so it dies with a SIGKILLed parent, and elsewhere it exits at its next step
1498
+ once its parent is gone.
1499
+ A file over either limit is recorded with error code `TIMEOUT` or
1500
+ `MEMORY_LIMIT` (the message names the running phase), stays unindexed, and is
1501
+ retried by the next sync; the rest of the run continues. `update` and `index` always print such a
1502
+ file in terminal output as `[CODE] <relPath>: <message>` (other per-file
1503
+ errors stay `--verbose` only), and write
1504
+ `Still converting <collection>/<relPath> after <n>s (budget <m>s)` to stderr
1505
+ once a file has been processing for 10 seconds or half its budget, whichever
1506
+ is sooner.
1507
+ `gno status` lists stopped files under "Stopped at conversion budget" in
1508
+ terminal output (JSON output is unchanged).
1509
+
1475
1510
  File/export adapter iteration has a 60-second deadline in addition to the byte,
1476
1511
  record, metadata, and total-character caps. A partial import that retains valid
1477
1512
  siblings exits successfully but is never silent: terminal output includes the
@@ -2976,7 +3011,11 @@ misconfigured `findings` block (see [Daemon Mode](../docs/DAEMON.md#scheduled-fi
2976
3011
  The `embedding-fingerprint` check is additive doctor-only diagnostics. It uses
2977
3012
  the active embed model and stored vector dimensions to report the current
2978
3013
  freshness fingerprint, pending/stale chunks, legacy empty-fingerprint vectors,
2979
- and stored fingerprint groups. Stale, legacy, and mixed groups are warnings;
3014
+ and stored fingerprint groups. On an index with an activated vector partition
3015
+ for the embed model (every index `gno embed` wrote since 2.7), the groups are
3016
+ that partition with its current chunk count and there are no legacy vectors;
3017
+ legacy `content_vectors` groups are read only before any partition activates.
3018
+ Stale, legacy, and mixed groups are warnings;
2980
3019
  recover with `gno embed`, or `gno embed --force` if vectors still look stale.
2981
3020
 
2982
3021
  The additive `activation` object uses the same contract as `gno status` and
@@ -3988,7 +4027,10 @@ gno similar <doc> [-n, --limit <num>] [--threshold <num>] [--cross-collection] [
3988
4027
 
3989
4028
  - Finds documents semantically similar to the source document
3990
4029
  - Requires embeddings to be generated (`gno embed`)
3991
- - Uses average document embedding for comparison
4030
+ - Source vector: the stored vector of the document's first chunk (the
4031
+ lowest-seq chunk whose vector is current), read from the active vector
4032
+ partition. `gno_similar` (MCP), `GET /api/doc/:id/similar` (REST) and graph
4033
+ similarity edges use the same rule, so scores match across surfaces
3992
4034
  - By default, limits results to same collection
3993
4035
 
3994
4036
  **Output (JSON):**
@@ -4203,6 +4245,9 @@ gno changes --follow --jsonl [--cursor <cursor>] [--collection <name>]
4203
4245
  old/new identity and hash snapshots, normalized structural deltas, pagination,
4204
4246
  cursor-expiry, and retention-truncation disclosure.
4205
4247
  - The journal never returns source bodies.
4248
+ - Link additions and removals in `structureDelta.links` are recorded only for
4249
+ Markdown sources, matching link extraction; content changes of other
4250
+ documents (code, text, converted files, records) are still journaled.
4206
4251
 
4207
4252
  **Follow mode (`--follow --jsonl`)** streams journal events as they land and is
4208
4253
  the durable automation input for consumers that resume across restarts.
@@ -4444,6 +4489,21 @@ gno serve --detach --stop
4444
4489
 
4445
4490
  ---
4446
4491
 
4492
+ #### Config reload while running
4493
+
4494
+ A running `gno serve` or `gno daemon` re-reads its config file before each
4495
+ request (REST, Web UI, HTTP MCP, daemon status) and every 2 seconds, and adopts
4496
+ changes to `collections` and `contexts` made by the CLI (`gno collection
4497
+ add/remove`) or by hand: status, MCP `gno_status`, the watcher and search scope
4498
+ follow without a restart. A newly added collection is watched from then on; its
4499
+ existing files are indexed by `gno update` (or the next start's initial sync).
4500
+ An adoption that changes the collection set or policies moves the egress policy
4501
+ epoch as a REST collection change does. The checks are the sessions config
4502
+ refresh's: an unreadable config file answers an error (`500`, sessions code
4503
+ `SESSIONS_RUNTIME_FAILURE`), never the stale config, and a config bound to
4504
+ another index is refused. Other settings (models, gateway, sessions sources
4505
+ outside the sessions routes) keep their restart behaviour.
4506
+
4447
4507
  ### gno daemon
4448
4508
 
4449
4509
  Start a headless long-running watcher process for continuous indexing.
@@ -4483,6 +4543,8 @@ is blocked.
4483
4543
 
4484
4544
  - Opens DB once at startup
4485
4545
  - Loads config and requires at least one configured collection
4546
+ - Follows collection edits to the config file while running (see
4547
+ [Config reload](#config-reload-while-running))
4486
4548
  - Starts the same watcher + embed scheduler used by `gno serve`
4487
4549
  - Exact contained eligible paths always use targeted content-hash sync.
4488
4550
  Ambiguous temp/directory/missing-name/recursive-delete events use bounded,
package/spec/mcp.md CHANGED
@@ -2188,12 +2188,11 @@ Find semantically similar documents using vector embeddings.
2188
2188
 
2189
2189
  **Algorithm:**
2190
2190
 
2191
- 1. Get all chunks for the source document
2192
- 2. Retrieve each chunk's stored embedding from the active vector partition (legacy `content_vectors` only before any partition activates); no model is loaded
2193
- 3. Compute average embedding across all chunks
2194
- 4. Search for nearest neighbors using sqlite-vec
2195
- 5. Exclude self and filter by collection if not crossCollection
2196
- 6. Return top N similar documents with scores
2191
+ 1. Take the stored vector of the source document's first chunk (the lowest-seq chunk whose vector is current) from the active vector partition (legacy `content_vectors` only before any partition activates); no model is loaded. This is the same source vector `gno similar`, `GET /api/doc/:id/similar` and graph similarity edges use, so scores match across surfaces
2192
+ 2. Normalize it to unit length
2193
+ 3. Search for nearest neighbors using sqlite-vec
2194
+ 4. Exclude self and filter by collection if not crossCollection
2195
+ 5. Return top N similar documents with scores
2197
2196
 
2198
2197
  **Errors:**
2199
2198
 
@@ -16,6 +16,7 @@ import {
16
16
  resolveHttpGatewayConfig,
17
17
  } from "../../mcp/http-security";
18
18
  import { startBackgroundRuntime } from "../../serve/background-runtime";
19
+ import { refreshResidentConfig } from "../../serve/resident-request";
19
20
  import { handleResidentStatus, handleStatus } from "../../serve/routes/api";
20
21
  import { createMcpHttpGateway } from "../../serve/routes/mcp";
21
22
  import { SessionsError } from "../../sessions/types";
@@ -106,6 +107,8 @@ export const authorizeDaemonStatus = async (
106
107
  ): Promise<Response> => {
107
108
  const authorization = await gateway.security.authorize(request, server);
108
109
  if (!authorization.ok) return authorization.response;
110
+ const refreshFailure = await refreshResidentConfig(runtime);
111
+ if (refreshFailure) return refreshFailure;
109
112
  try {
110
113
  enforceCollectionEgress({
111
114
  collections: runtime.config.collections,
@@ -132,13 +135,15 @@ export const authorizeDaemonStatus = async (
132
135
  }
133
136
  };
134
137
 
135
- export function handleDaemonAppStatus(
138
+ export async function handleDaemonAppStatus(
136
139
  runtime: ResidentRuntime,
137
140
  host: string
138
- ): Promise<Response> | Response {
141
+ ): Promise<Response> {
139
142
  if (!isHttpGatewayLoopbackBind(host)) {
140
143
  return new Response(null, { status: 404 });
141
144
  }
145
+ const refreshFailure = await refreshResidentConfig(runtime);
146
+ if (refreshFailure) return refreshFailure;
142
147
  return handleStatus(runtime.ctxHolder.current, {
143
148
  getResidentStatus: () => runtime.getStatus(),
144
149
  });
@@ -31,6 +31,7 @@ import {
31
31
  resolveFindingsSchedule,
32
32
  } from "../../core/findings-run-state";
33
33
  import { formatVectorPartitionLines } from "../../core/vector-partition-status";
34
+ import { getEmbeddingFingerprint } from "../../embed/fingerprint";
34
35
  import { getCodeChunkingStatus } from "../../ingestion/chunker";
35
36
  import { ModelCache } from "../../llm/cache";
36
37
  import { getActivePreset, resolveModelUri } from "../../llm/registry";
@@ -42,6 +43,8 @@ import {
42
43
  getLoadAttempts,
43
44
  } from "../../store/sqlite/setup";
44
45
  import { getStoredEmbeddingFingerprint } from "../../store/vector/freshness";
46
+ import { currentOwnerCount } from "../../store/vector/runtime-compat";
47
+ import { storedVectorPartition } from "../../store/vector/status";
45
48
  import {
46
49
  buildDoctorActivation,
47
50
  checkConnectorActivation,
@@ -319,10 +322,14 @@ async function checkEmbeddingFingerprints(
319
322
  ];
320
323
  }
321
324
 
322
- const legacyChunks =
323
- db
324
- .query<{ count: number }, [string]>(
325
- `
325
+ // An index embedded since 2.7 keeps its vectors in the active partition;
326
+ // legacy `content_vectors` count only before any partition activates.
327
+ const partition = storedVectorPartition(db, model);
328
+ const legacyChunks = partition
329
+ ? 0
330
+ : (db
331
+ .query<{ count: number }, [string]>(
332
+ `
326
333
  SELECT COUNT(*) as count
327
334
  FROM content_vectors v
328
335
  JOIN content_chunks c
@@ -336,12 +343,23 @@ async function checkEmbeddingFingerprints(
336
343
  AND d.active = 1
337
344
  )
338
345
  `
339
- )
340
- .get(model)?.count ?? 0;
341
-
342
- const groups = db
343
- .query<{ model: string; fingerprint: string; count: number }, []>(
344
- `
346
+ )
347
+ .get(model)?.count ?? 0);
348
+
349
+ const groups = partition
350
+ ? [
351
+ {
352
+ model,
353
+ fingerprint: getEmbeddingFingerprint({
354
+ modelUri: model,
355
+ dimensions: partition.dimensions,
356
+ }),
357
+ count: currentOwnerCount(db, partition.partitionId),
358
+ },
359
+ ]
360
+ : db
361
+ .query<{ model: string; fingerprint: string; count: number }, []>(
362
+ `
345
363
  SELECT
346
364
  v.model as model,
347
365
  v.embed_fingerprint as fingerprint,
@@ -358,18 +376,18 @@ async function checkEmbeddingFingerprints(
358
376
  GROUP BY v.model, v.embed_fingerprint
359
377
  ORDER BY count DESC, v.model ASC, v.embed_fingerprint ASC
360
378
  `
361
- )
362
- .all()
363
- .map((group) => ({
364
- model: group.model,
365
- fingerprint: group.fingerprint,
366
- count: group.count,
367
- current:
368
- group.model === model && group.fingerprint === currentFingerprint,
369
- legacy: group.fingerprint === "",
370
- }));
371
-
372
- const mixedGroups = groups.length;
379
+ )
380
+ .all();
381
+ const fingerprintGroups = groups.map((group) => ({
382
+ model: group.model,
383
+ fingerprint: group.fingerprint,
384
+ count: group.count,
385
+ current:
386
+ group.model === model && group.fingerprint === currentFingerprint,
387
+ legacy: group.fingerprint === "",
388
+ }));
389
+
390
+ const mixedGroups = fingerprintGroups.length;
373
391
  const pendingChunks = statusResult.value.embeddingBacklog;
374
392
  const hasWarnings =
375
393
  pendingChunks > 0 || legacyChunks > 0 || mixedGroups > 1;
@@ -380,7 +398,7 @@ async function checkEmbeddingFingerprints(
380
398
  pendingChunks,
381
399
  legacyChunks,
382
400
  mixedGroups,
383
- groups,
401
+ groups: fingerprintGroups,
384
402
  };
385
403
 
386
404
  const message =
@@ -392,7 +410,7 @@ async function checkEmbeddingFingerprints(
392
410
  if (hasWarnings) {
393
411
  details.push("Run: gno embed");
394
412
  details.push("If vectors still look stale, run: gno embed --force");
395
- for (const group of groups) {
413
+ for (const group of fingerprintGroups) {
396
414
  const label = describeFingerprintGroup(group);
397
415
  const fingerprint = group.legacy
398
416
  ? "(empty)"
@@ -28,7 +28,11 @@ import {
28
28
  withContentTypeRules,
29
29
  } from "../../ingestion";
30
30
  import { type EmbedResult, embedStageOutcome } from "./embed";
31
- import { formatSyncResultLines, initStore } from "./shared";
31
+ import {
32
+ formatSyncResultLines,
33
+ initStore,
34
+ reportSlowConversion,
35
+ } from "./shared";
32
36
 
33
37
  /**
34
38
  * Options for index command.
@@ -202,7 +206,11 @@ export async function index(options: IndexOptions = {}): Promise<IndexResult> {
202
206
  collections,
203
207
  store,
204
208
  withContentTypeRules(
205
- { gitPull: options.gitPull, runUpdateCmd: true },
209
+ {
210
+ gitPull: options.gitPull,
211
+ runUpdateCmd: true,
212
+ onSlowConversion: reportSlowConversion,
213
+ },
206
214
  config
207
215
  )
208
216
  );
@@ -587,21 +587,17 @@ export async function similar(
587
587
  }
588
588
  const db = store.getRawDb();
589
589
 
590
- // Stored vector of the document's first chunk, from the active partition
590
+ // Similarity source vector (first chunk, unit length), active partition
591
591
  const {
592
- readStoredDocumentVectors,
592
+ readSimilaritySourceVectors,
593
593
  resolveStoredVectorSource,
594
594
  similarityHitDocuments,
595
595
  storedVectorSearchOptions,
596
596
  } = await import("../../store/vector/stored-vectors.js");
597
597
  const source = resolveStoredVectorSource(db, modelPreset.embed);
598
- const [embedding] =
599
- readStoredDocumentVectors(
600
- db,
601
- source,
602
- [{ id: doc.id, mirrorHash: doc.mirrorHash }],
603
- { firstChunkOnly: true }
604
- ).get(doc.id) ?? [];
598
+ const embedding = readSimilaritySourceVectors(db, source, [
599
+ { id: doc.id, mirrorHash: doc.mirrorHash },
600
+ ]).get(doc.id);
605
601
 
606
602
  if (!embedding) {
607
603
  return {
@@ -610,20 +606,7 @@ export async function similar(
610
606
  isValidation: true,
611
607
  };
612
608
  }
613
-
614
- // Normalize embedding for cosine similarity
615
609
  const dimensions = embedding.length;
616
- let norm = 0;
617
- for (let i = 0; i < dimensions; i++) {
618
- const val = embedding[i] ?? 0;
619
- norm += val * val;
620
- }
621
- norm = Math.sqrt(norm);
622
- if (norm > 0) {
623
- for (let i = 0; i < dimensions; i++) {
624
- embedding[i] = (embedding[i] ?? 0) / norm;
625
- }
626
- }
627
610
 
628
611
  // Create vector index port for search
629
612
  const { createVectorIndexPort } =
@@ -6,7 +6,7 @@
6
6
  */
7
7
 
8
8
  import type { Collection, Config } from "../../config/types";
9
- import type { SyncResult } from "../../ingestion";
9
+ import type { SlowConversionEvent, SyncResult } from "../../ingestion";
10
10
  import type { SearchResults } from "../../pipeline/types";
11
11
 
12
12
  import {
@@ -20,6 +20,7 @@ import {
20
20
  loadConfig,
21
21
  writeConfigWarningsToStderr,
22
22
  } from "../../config";
23
+ import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
23
24
  import { SqliteAdapter } from "../../store/sqlite/adapter";
24
25
  import { assertCliSessionBinding } from "../session-binding";
25
26
 
@@ -164,6 +165,21 @@ export function decorateSearchResultsForIndex(
164
165
  };
165
166
  }
166
167
 
168
+ /**
169
+ * Name a file whose conversion is still running past the notice threshold,
170
+ * so an operator can see which file a slow run is working on (stderr, so
171
+ * JSON output on stdout stays parseable).
172
+ */
173
+ export function reportSlowConversion(event: SlowConversionEvent): void {
174
+ process.stderr.write(
175
+ `Still converting ${event.collection}/${event.relPath} after ${formatSeconds(event.elapsedMs)} (budget ${formatSeconds(event.budgetMs)})\n`
176
+ );
177
+ }
178
+
179
+ /** Seconds with one decimal below ten seconds, whole seconds above. */
180
+ const formatSeconds = (ms: number): string =>
181
+ ms < 10_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms / 1000)}s`;
182
+
167
183
  /**
168
184
  * Format sync result lines (shared between update and index commands).
169
185
  */
@@ -194,6 +210,15 @@ export function formatSyncResultLines(
194
210
  if (c.filesErrored > 0) {
195
211
  lines.push(` ${c.filesErrored} errors`);
196
212
  }
213
+ // A file stopped at its conversion budget is always named, not only in
214
+ // verbose output: it stays unindexed until a later run finishes it.
215
+ if (!options.verbose) {
216
+ for (const err of c.errors) {
217
+ if (BUDGET_ERROR_CODES.has(err.code)) {
218
+ lines.push(` [${err.code}] ${err.relPath}: ${err.message}`);
219
+ }
220
+ }
221
+ }
197
222
  if (c.filesMarkedInactive > 0) {
198
223
  lines.push(` ${c.filesMarkedInactive} marked inactive`);
199
224
  }
@@ -27,6 +27,7 @@ import {
27
27
  formatMemoryStatusLines,
28
28
  } from "../../core/memory-diagnostics";
29
29
  import { formatVectorPartitionLines } from "../../core/vector-partition-status";
30
+ import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
30
31
  import { ModelCache } from "../../llm/cache";
31
32
  import { getActivePreset, resolveModelUri } from "../../llm/registry";
32
33
  import { getConnectorVerificationTargets } from "../../serve/connectors";
@@ -64,9 +65,56 @@ export type StatusResult =
64
65
  contentTypeBoost: ContentTypeBoostStatus;
65
66
  memory: MemoryStatus;
66
67
  backgroundIssues: ResidentBackgroundIssue[];
68
+ /** Files whose latest conversion stopped at the per-file budget. */
69
+ budgetStops: BudgetStop[];
67
70
  }
68
71
  | { success: false; error: string };
69
72
 
73
+ /** A file left unindexed because its conversion overran its budget. */
74
+ export interface BudgetStop {
75
+ collection: string;
76
+ relPath: string;
77
+ code: string;
78
+ message: string;
79
+ }
80
+
81
+ /** Most recent ingest errors inspected for budget stops. */
82
+ const BUDGET_STOP_SCAN_LIMIT = 100;
83
+ /** Most budget stops listed in status output. */
84
+ const MAX_BUDGET_STOPS = 10;
85
+
86
+ /**
87
+ * Files whose latest indexing attempt stopped at the conversion budget, from
88
+ * the recent error log, kept only while the document still carries that code.
89
+ */
90
+ async function collectBudgetStops(store: SqliteAdapter): Promise<BudgetStop[]> {
91
+ const recent = await store.getRecentErrors(BUDGET_STOP_SCAN_LIMIT);
92
+ if (!recent.ok) return [];
93
+ const stops: BudgetStop[] = [];
94
+ const seen = new Set<string>();
95
+ for (const row of recent.value) {
96
+ const key = `${row.collection}\u0000${row.relPath}`;
97
+ if (seen.has(key)) continue;
98
+ seen.add(key);
99
+ if (!BUDGET_ERROR_CODES.has(row.code)) continue;
100
+ const document = await store.getDocument(row.collection, row.relPath);
101
+ if (
102
+ document.ok &&
103
+ document.value?.active &&
104
+ document.value.lastErrorCode === row.code
105
+ ) {
106
+ stops.push({
107
+ collection: row.collection,
108
+ relPath: row.relPath,
109
+ code: row.code,
110
+ message: row.message,
111
+ });
112
+ if (stops.length >= MAX_BUDGET_STOPS) break;
113
+ }
114
+ }
115
+ return stops;
116
+ }
117
+
70
118
  /** A background issue reported by (or about) a running detached resident. */
71
119
  export type ResidentBackgroundIssue = BackgroundIssue & {
72
120
  process: "serve" | "daemon";
@@ -368,6 +416,7 @@ export async function status(
368
416
  contentTypeBoost: buildContentTypeBoostStatus(config.contentTypes ?? []),
369
417
  memory: await buildMemoryStatus(store, config.collections),
370
418
  backgroundIssues: await collectResidentIssues(),
419
+ budgetStops: await collectBudgetStops(store),
371
420
  };
372
421
  } finally {
373
422
  await store.close();
@@ -439,6 +488,10 @@ export function formatStatus(
439
488
  (issue) =>
440
489
  ` ${issue.process} (pid ${issue.pid}): ${formatBackgroundIssue(issue)}`
441
490
  );
491
+ const stopLines = result.budgetStops.map(
492
+ (stop) =>
493
+ ` ${stop.collection}/${stop.relPath} [${stop.code}] ${stop.message}`
494
+ );
442
495
  return [
443
496
  formatTerminal(
444
497
  result.status,
@@ -446,6 +499,13 @@ export function formatStatus(
446
499
  result.contentTypeBoost,
447
500
  result.memory
448
501
  ),
502
+ ...(stopLines.length
503
+ ? [
504
+ "",
505
+ "Stopped at conversion budget (retried by the next update):",
506
+ ...stopLines,
507
+ ]
508
+ : []),
449
509
  ...(issueLines.length ? ["", "Background issues:", ...issueLines] : []),
450
510
  ].join("\n");
451
511
  }
@@ -15,7 +15,11 @@ import {
15
15
  type SyncResult,
16
16
  withContentTypeRules,
17
17
  } from "../../ingestion";
18
- import { formatSyncResultLines, initStore } from "./shared";
18
+ import {
19
+ formatSyncResultLines,
20
+ initStore,
21
+ reportSlowConversion,
22
+ } from "./shared";
19
23
 
20
24
  /**
21
25
  * Options for update command.
@@ -66,6 +70,7 @@ export async function update(
66
70
  {
67
71
  gitPull: options.gitPull,
68
72
  runUpdateCmd: true,
73
+ onSlowConversion: reportSlowConversion,
69
74
  },
70
75
  config
71
76
  )
@@ -13,10 +13,16 @@ import {
13
13
  createVectorIndexPort,
14
14
  createVectorStatsPort,
15
15
  } from "../../store/vector";
16
+ import {
17
+ rebuildPartitionIndex,
18
+ syncPartitionIndex,
19
+ } from "../../store/vector/partition-index";
16
20
  import {
17
21
  dropVectorPartition,
22
+ storedVectorPartition,
18
23
  type VectorPartitionStatus,
19
24
  } from "../../store/vector/status";
25
+ import { loadSqliteVec } from "../../store/vector/variants";
20
26
 
21
27
  // ─────────────────────────────────────────────────────────────────────────────
22
28
  // Types
@@ -76,7 +82,9 @@ function inferDimensions(
76
82
  // ─────────────────────────────────────────────────────────────────────────────
77
83
 
78
84
  /**
79
- * Sync vec0 index with content_vectors (add missing, remove orphans).
85
+ * Sync the vec0 index with stored vectors (add missing, remove orphans): the
86
+ * active vector partition's variants, or legacy `content_vectors` on an
87
+ * index that never activated a partition.
80
88
  */
81
89
  export async function vecSync(
82
90
  options: VecOptions = {}
@@ -112,6 +120,21 @@ export async function vecSync(
112
120
  try {
113
121
  const db = store.getRawDb();
114
122
 
123
+ const partition = storedVectorPartition(db, modelUri);
124
+ if (partition) {
125
+ if (!(await loadSqliteVec(db))) {
126
+ return {
127
+ success: false,
128
+ error: "sqlite-vec not available. Cannot sync index.",
129
+ };
130
+ }
131
+ return {
132
+ success: true,
133
+ ...syncPartitionIndex(db, partition),
134
+ model: modelUri,
135
+ };
136
+ }
137
+
115
138
  // Infer dimensions from stored vectors for this model
116
139
  const dimensions = inferDimensions(db, modelUri);
117
140
  if (dimensions === null) {
@@ -154,7 +177,9 @@ export async function vecSync(
154
177
  }
155
178
 
156
179
  /**
157
- * Rebuild vec0 index from content_vectors (drop + recreate + repopulate).
180
+ * Rebuild the vec0 index (drop + recreate + repopulate) from the active
181
+ * vector partition's variants, or from legacy `content_vectors` on an index
182
+ * that never activated a partition.
158
183
  */
159
184
  export async function vecRebuild(
160
185
  options: VecOptions = {}
@@ -190,6 +215,21 @@ export async function vecRebuild(
190
215
  try {
191
216
  const db = store.getRawDb();
192
217
 
218
+ const partition = storedVectorPartition(db, modelUri);
219
+ if (partition) {
220
+ if (!(await loadSqliteVec(db))) {
221
+ return {
222
+ success: false,
223
+ error: "sqlite-vec not available. Cannot rebuild index.",
224
+ };
225
+ }
226
+ return {
227
+ success: true,
228
+ count: rebuildPartitionIndex(db, partition),
229
+ model: modelUri,
230
+ };
231
+ }
232
+
193
233
  // Infer dimensions from stored vectors for this model
194
234
  const dimensions = inferDimensions(db, modelUri);
195
235
  if (dimensions === null) {