@gmickel/gno 2.2.1 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -4
- package/assets/skill/README.md +60 -21
- package/assets/skill/SKILL.md +7 -0
- package/assets/skill/cli-reference.md +16 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.2.1.zip → gno-browser-clipper-v2.3.1.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.3.1.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +21 -0
- package/spec/mcp.md +9 -0
- package/spec/output-schemas/status.schema.json +41 -0
- package/src/cli/commands/shared.ts +6 -0
- package/src/cli/commands/status.ts +9 -0
- package/src/config/chunking.ts +46 -0
- package/src/config/types.ts +6 -1
- package/src/core/chunking-status.ts +30 -0
- package/src/core/mutation-generations.ts +2 -0
- package/src/ingestion/chunking.ts +108 -0
- package/src/ingestion/record-container.ts +9 -8
- package/src/ingestion/sync-options.ts +3 -1
- package/src/ingestion/sync.ts +33 -24
- package/src/ingestion/types.ts +15 -2
- package/src/mcp/tools/status.ts +5 -0
- package/src/sdk/client.ts +1 -0
- package/src/serve/status-model.ts +2 -0
- package/src/serve/status.ts +2 -0
- package/src/store/chunking.ts +31 -0
- package/src/store/sqlite/adapter.ts +104 -2
- package/src/store/sqlite/chunking-policy.ts +212 -0
- package/src/store/types.ts +32 -1
- package/browser-extension/artifacts/gno-browser-clipper-v2.2.1.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -117,7 +117,7 @@ gno daemon --detach # headless indexing + resident MCP gateway
|
|
|
117
117
|
|
|
118
118
|
<!-- public-truth:current-version -->
|
|
119
119
|
|
|
120
|
-
> Current source version: **v2.
|
|
120
|
+
> Current source version: **v2.3.1**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
121
121
|
|
|
122
122
|
<!-- /public-truth -->
|
|
123
123
|
|
|
@@ -288,6 +288,9 @@ gno query "ECONNREFUSED 127.0.0.1:5432" --thorough
|
|
|
288
288
|
|
|
289
289
|
```bash
|
|
290
290
|
gno setup ~/notes --name notes # Build BM25 and prove an exact local result
|
|
291
|
+
gno skill install --target claude --scope user # Choose your agent
|
|
292
|
+
gno agents install --target claude # Teach retrieval and writing discipline
|
|
293
|
+
gno agents verify --target claude
|
|
291
294
|
gno daemon --detach # Keep index fresh in the background (macOS/Linux)
|
|
292
295
|
gno query "auth best practices" # Hybrid search
|
|
293
296
|
gno ask "summarize the API" --answer # AI answer with citations
|
|
@@ -365,6 +368,30 @@ See also: [docs/DAEMON.md](./docs/DAEMON.md)
|
|
|
365
368
|
|
|
366
369
|
### Connect to AI Agents
|
|
367
370
|
|
|
371
|
+
For a second brain or LLM wiki, install both a connector and the retrieval
|
|
372
|
+
protocol. The skill teaches GNO commands and workflows; the instruction block
|
|
373
|
+
teaches the harness when to retrieve, how to choose an efficient search path,
|
|
374
|
+
and how to cite and maintain knowledge. A connector alone does not establish
|
|
375
|
+
that discipline.
|
|
376
|
+
|
|
377
|
+
```bash
|
|
378
|
+
gno skill install --target claude --scope user
|
|
379
|
+
gno agents install --target claude
|
|
380
|
+
gno agents verify --target claude
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
Choose your harness target (`claude`, `codex`, `opencode`, `openclaw`, or
|
|
384
|
+
`hermes`). For MCP clients, install the MCP connector instead of the skill,
|
|
385
|
+
then add the protocol where the harness supports it. `gno agents install`
|
|
386
|
+
without a target covers detected supported harnesses; see the
|
|
387
|
+
[agent instructions guide](docs/AGENT-INSTRUCTIONS.md) for the target matrix and
|
|
388
|
+
manual guidance. It preserves text outside its managed block.
|
|
389
|
+
|
|
390
|
+
Start a fresh agent session and ask a question answered by an indexed document.
|
|
391
|
+
Check that the agent retrieves it and cites the source. `gno agents verify`
|
|
392
|
+
checks the installed block, not model behavior. CLI and Web UI users can skip
|
|
393
|
+
agent setup.
|
|
394
|
+
|
|
368
395
|
#### MCP Server (Claude Desktop, Cursor, Zed, etc.)
|
|
369
396
|
|
|
370
397
|
One command to add GNO to your AI assistant:
|
|
@@ -391,7 +418,7 @@ target, add `--force` to preview the replacement without writing it.
|
|
|
391
418
|
|
|
392
419
|
Check status: `gno mcp status`
|
|
393
420
|
|
|
394
|
-
#### Skills (Claude Code, Codex, OpenCode, OpenClaw)
|
|
421
|
+
#### Skills (Claude Code, Codex, OpenCode, OpenClaw, Hermes)
|
|
395
422
|
|
|
396
423
|
Skills integrate via CLI with no MCP overhead and include second-brain recipe playbooks:
|
|
397
424
|
|
|
@@ -400,6 +427,7 @@ gno skill install --scope user # User-wide
|
|
|
400
427
|
gno skill install --target codex # Codex
|
|
401
428
|
gno skill install --target opencode # OpenCode
|
|
402
429
|
gno skill install --target openclaw # OpenClaw
|
|
430
|
+
gno skill install --target hermes # Hermes
|
|
403
431
|
gno skill install --target all # All targets
|
|
404
432
|
```
|
|
405
433
|
|
|
@@ -627,10 +655,12 @@ Give your local LLM agents a long-term memory. GNO integrates as a Claude Code s
|
|
|
627
655
|
|
|
628
656
|
### Skills
|
|
629
657
|
|
|
630
|
-
Skills
|
|
658
|
+
Skills teach Claude Code, Codex, OpenCode, OpenClaw, and Hermes how to use GNO. Pair the skill with the retrieval protocol for second-brain and LLM-wiki work:
|
|
631
659
|
|
|
632
660
|
```bash
|
|
633
661
|
gno skill install --scope user
|
|
662
|
+
gno agents install --target claude
|
|
663
|
+
gno agents verify --target claude
|
|
634
664
|
```
|
|
635
665
|
|
|
636
666
|

|
|
@@ -818,7 +848,23 @@ On [Omarchy](https://omarchy.org/), [**GNO Recall**](https://github.com/gmickel/
|
|
|
818
848
|
omarchy plugin add https://github.com/gmickel/omarchy-gno-recall --enable
|
|
819
849
|
```
|
|
820
850
|
|
|
821
|
-
|
|
851
|
+
Then install the plugin's verified runtime and, optionally, the Super+R binding:
|
|
852
|
+
|
|
853
|
+
```bash
|
|
854
|
+
cd ~/.config/omarchy/plugins/gmickel.gno-recall
|
|
855
|
+
./scripts/install-runtime.sh
|
|
856
|
+
./scripts/install-keybind.sh # Optional; checks for conflicts
|
|
857
|
+
```
|
|
858
|
+
|
|
859
|
+
[Marketplace listing](https://omarchyplugins.com/plugin.html?id=gmickel.gno-recall)
|
|
860
|
+
· [Setup and runtime compatibility](https://github.com/gmickel/omarchy-gno-recall#install)
|
|
861
|
+
· [Screenshots](https://github.com/gmickel/omarchy-gno-recall#screenshots)
|
|
862
|
+
|
|
863
|
+
Recall uses its own verified GNO/Bun runtime. A global GNO upgrade does not
|
|
864
|
+
update it; follow the plugin's update instructions and shared-index
|
|
865
|
+
compatibility guidance.
|
|
866
|
+
|
|
867
|
+

|
|
822
868
|
|
|
823
869
|
---
|
|
824
870
|
|
package/assets/skill/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# GNO — Agent Skill
|
|
2
2
|
|
|
3
3
|
Local-first semantic search for documents, notes, and knowledge bases,
|
|
4
|
-
packaged as a Claude Code / Codex / OpenCode / OpenClaw skill.
|
|
4
|
+
packaged as a Claude Code / Codex / OpenCode / OpenClaw / Hermes skill.
|
|
5
5
|
|
|
6
6
|
> **TL;DR** — this folder is a runnable agent skill. Drop it into any
|
|
7
|
-
> Claude Code, Codex, OpenCode, or
|
|
7
|
+
> Claude Code, Codex, OpenCode, OpenClaw, or Hermes workspace and the agent can
|
|
8
8
|
> index and search your local files through the `gno` CLI.
|
|
9
9
|
|
|
10
10
|
## Prerequisites
|
|
@@ -13,14 +13,10 @@ You need the `gno` CLI installed locally. The skill drives it; it does
|
|
|
13
13
|
not ship the binary itself.
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
|
-
|
|
17
|
-
curl -fsSL https://gno.sh/install | bash
|
|
18
|
-
|
|
19
|
-
# npm / Bun
|
|
20
|
-
bun add -g @gmickel/gno # or: npm install -g @gmickel/gno
|
|
16
|
+
bun install -g @gmickel/gno
|
|
21
17
|
```
|
|
22
18
|
|
|
23
|
-
Verify
|
|
19
|
+
Verify the installed release with `gno --version`. Use the bundled skill to match that release.
|
|
24
20
|
|
|
25
21
|
## Install the skill
|
|
26
22
|
|
|
@@ -30,8 +26,7 @@ you're starting from.
|
|
|
30
26
|
### Option A — install from your local `gno` (recommended if you already have GNO)
|
|
31
27
|
|
|
32
28
|
If you already installed GNO, it ships this skill in-tree. One command
|
|
33
|
-
drops it into the right place for every supported agent,
|
|
34
|
-
with your installed GNO version:
|
|
29
|
+
drops it into the right place for every supported agent, matching the GNO version installed when you run it:
|
|
35
30
|
|
|
36
31
|
```bash
|
|
37
32
|
gno skill install --target claude # Claude Code (default)
|
|
@@ -59,23 +54,67 @@ gno skill paths # resolved installation paths
|
|
|
59
54
|
gno skill uninstall --target all # remove from all agents
|
|
60
55
|
```
|
|
61
56
|
|
|
62
|
-
|
|
63
|
-
|
|
57
|
+
After upgrading GNO, refresh the installed copy as described below.
|
|
58
|
+
|
|
59
|
+
### Option B: ClawHub-managed installation
|
|
60
|
+
|
|
61
|
+
Use the [GNO listing](https://clawhub.ai/gmickel/gno) when your harness manages
|
|
62
|
+
skills through a registry. The GNO executable is still required separately.
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
# OpenClaw workspace: run from its workspace root
|
|
66
|
+
bunx clawhub@0.23.3 install @gmickel/gno
|
|
67
|
+
bunx clawhub@0.23.3 update @gmickel/gno
|
|
68
|
+
|
|
69
|
+
# Hermes hub installation
|
|
70
|
+
hermes skills install clawhub/gno # May require review; prefer the local installer
|
|
71
|
+
hermes skills check
|
|
72
|
+
hermes skills update gno
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
GNO releases publish the same source bundle to ClawHub. Choose one installer
|
|
76
|
+
for a given skill directory so registry and local copies do not overwrite one
|
|
77
|
+
another. For Hermes, prefer the local GNO installer if your hub version cannot
|
|
78
|
+
retrieve the full bundle. A bare SKILL.md URL may omit recipes and references.
|
|
64
79
|
|
|
65
|
-
|
|
80
|
+
## Install the retrieval protocol too
|
|
66
81
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
82
|
+
For second-brain and LLM-wiki work, pair the skill with standing instructions
|
|
83
|
+
for the harness. The skill teaches commands and workflows; the protocol teaches
|
|
84
|
+
when to retrieve, which search path to use, and how to cite and maintain knowledge.
|
|
70
85
|
|
|
71
86
|
```bash
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
openclaw skills info gno # inspect what's installed
|
|
87
|
+
gno agents install # Detected supported harnesses
|
|
88
|
+
gno agents verify
|
|
75
89
|
```
|
|
76
90
|
|
|
77
|
-
|
|
78
|
-
|
|
91
|
+
Use `--target claude`, `codex`, `opencode`, `openclaw`, or `hermes` for one
|
|
92
|
+
harness. MCP users should install the protocol too where supported. Text outside
|
|
93
|
+
the managed instruction block stays unchanged. Start a fresh session and verify
|
|
94
|
+
that the agent retrieves and cites a document from your index. The verify command
|
|
95
|
+
checks the block, not model behavior. See [agent instructions](https://gno.sh/docs/agents-install).
|
|
96
|
+
|
|
97
|
+
## Keep installations current
|
|
98
|
+
|
|
99
|
+
Local installs copy the skill from your installed GNO release. After upgrading,
|
|
100
|
+
refresh the selected target and protocol:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
gno skill install --target hermes --scope user --force
|
|
104
|
+
gno agents update --target hermes
|
|
105
|
+
gno agents verify --target hermes
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
`--force` replaces local skill edits. Preserve your customizations before using
|
|
109
|
+
it. Change the target for another harness. Registry installs use their native
|
|
110
|
+
update command instead; upgrading the GNO executable alone does not refresh them.
|
|
111
|
+
|
|
112
|
+
For unattended registry updates, schedule only the chosen skill's native update
|
|
113
|
+
command with your OS scheduler, using the same user, workspace, and executable
|
|
114
|
+
paths as the interactive install. Preserve logs and keep safety prompts enabled;
|
|
115
|
+
an update that needs approval must remain pending. Run `gno agents update` after
|
|
116
|
+
upgrading GNO. New skill versions can require a newer executable, so update GNO
|
|
117
|
+
before refreshing the skill. Publishing a release does not modify users' machines.
|
|
79
118
|
|
|
80
119
|
## What the skill teaches the agent
|
|
81
120
|
|
package/assets/skill/SKILL.md
CHANGED
|
@@ -509,6 +509,13 @@ gno embed --collection travel
|
|
|
509
509
|
|
|
510
510
|
MCP `gno.sync` and `gno.capture` do NOT auto-embed. Use CLI for embedding.
|
|
511
511
|
|
|
512
|
+
Optional index-wide YAML `chunking.maxTokens` and `chunking.overlapPercent`
|
|
513
|
+
control size and overlap (`0.15` means 15%). Leave defaults unless asked to
|
|
514
|
+
tune them. After a change, `gno index` rechunks cached mirrors and embeds;
|
|
515
|
+
`gno update` rechunks only. Check `gno status --json` fields
|
|
516
|
+
`chunking.pendingMirrors` and `embeddingBacklog`. Use separate `--config`
|
|
517
|
+
files and `--index` names for comparisons; see [cli-reference.md](cli-reference.md).
|
|
518
|
+
|
|
512
519
|
## Capture Notes
|
|
513
520
|
|
|
514
521
|
Use `gno capture` for quick second-brain writes into an editable collection:
|
|
@@ -24,6 +24,22 @@ punctuation. NFC/case-equivalent spellings share one identity. See
|
|
|
24
24
|
|
|
25
25
|
## Initialization
|
|
26
26
|
|
|
27
|
+
### Optional index-wide chunking
|
|
28
|
+
|
|
29
|
+
User YAML can set `chunking: { maxTokens: 256, overlapPercent: 0.15 }`.
|
|
30
|
+
Defaults remain 800 approximate tokens and 0.15 overlap; unchanged/default
|
|
31
|
+
config does not force a rebuild. The token estimate uses four characters per
|
|
32
|
+
token, and overlap accepts a fraction from 0 through 0.5.
|
|
33
|
+
|
|
34
|
+
After changing policy, `gno index` rechunks cached mirrors and embeds changed
|
|
35
|
+
chunks. `gno update` performs the rechunking without embedding. The policy
|
|
36
|
+
applies across the index even for targeted source sync; original files are not
|
|
37
|
+
read merely to rechunk. `gno status --json` reports configured/applied policy,
|
|
38
|
+
pending mirrors/documents, and a separate embedding backlog. On
|
|
39
|
+
`CHUNKING_POLICY_CONFLICT`, reopen the client with the intended configuration.
|
|
40
|
+
Compare policies with separate `--config` files and `--index` names, holding
|
|
41
|
+
corpus, models, query set, and type boosts constant.
|
|
42
|
+
|
|
27
43
|
### gno setup
|
|
28
44
|
|
|
29
45
|
Preferred activation path: add one folder, prove a real exact lexical result,
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
674db0a56d6ba5a485f4d6195066a7cd700b3dbbbcbf921b12047548fcdd21e0 gno-browser-clipper-v2.3.1.zip
|
package/package.json
CHANGED
package/spec/cli.md
CHANGED
|
@@ -531,6 +531,27 @@ protection remain authoritative.
|
|
|
531
531
|
`graphHints` is active: ordered hints type
|
|
532
532
|
projected wiki/markdown edges and surface in graph traversal/diagnose metadata.
|
|
533
533
|
|
|
534
|
+
Optional root `chunking` sets one policy per index. `maxTokens` defaults to 800
|
|
535
|
+
and accepts integers from 10 through 2251799813685247; the chunker estimates
|
|
536
|
+
four characters per token. `overlapPercent` defaults to 0.15 and accepts a
|
|
537
|
+
fraction from 0 through 0.5. Missing fields inherit defaults. Invalid values
|
|
538
|
+
fail configuration validation before index mutation.
|
|
539
|
+
|
|
540
|
+
Omitted or explicitly default settings preserve existing chunks and embeddings
|
|
541
|
+
on upgrade. A policy change rechunks cached Markdown mirrors during the next
|
|
542
|
+
index/update or targeted sync, including mirrors outside the source-refresh
|
|
543
|
+
collection. Rechunking does not read original files or claim source freshness.
|
|
544
|
+
Sync-only operations leave changed chunks pending embedding; `gno index`
|
|
545
|
+
also embeds. A stale writer receives `CHUNKING_POLICY_CONFLICT` and must reopen
|
|
546
|
+
its client/runtime with the intended configuration.
|
|
547
|
+
|
|
548
|
+
Status JSON adds `chunking` with `configured` parameters, `applied` parameters
|
|
549
|
+
(null for empty/mixed layouts), `state` (`empty`, `legacy-default`, `current`,
|
|
550
|
+
`pending`, or `mixed`), `pendingDocuments`, and `pendingMirrors`. These counts
|
|
551
|
+
cover active cached content, independently of source errors and embedding
|
|
552
|
+
backlog. Sync receipts optionally add `rechunkedMirrors` when cached layouts
|
|
553
|
+
were updated; existing file counters keep their source-refresh meaning.
|
|
554
|
+
|
|
534
555
|
`collections[].sourceAvailability` is optional; omitted means `any`. Exact
|
|
535
556
|
values: `any` | `local`. There is no separate public knob beyond these two
|
|
536
557
|
modes and no claim that availability is egress policy. `any` preserves
|
package/spec/mcp.md
CHANGED
|
@@ -1352,6 +1352,15 @@ counters; it never claims attachment to another process.
|
|
|
1352
1352
|
normalized IDs/factors plus the rules fingerprint; path prefixes are never
|
|
1353
1353
|
returned.
|
|
1354
1354
|
|
|
1355
|
+
`chunking` reports the index-wide configuration and cached layout readiness:
|
|
1356
|
+
`configured` and `applied` contain `maxTokens` and `overlapPercent` (`applied`
|
|
1357
|
+
is null for empty/mixed layouts); `state` is `empty`, `legacy-default`,
|
|
1358
|
+
`current`, `pending`, or `mixed`; `pendingDocuments` and `pendingMirrors` count
|
|
1359
|
+
active cached content using another policy. Source refresh errors and embedding
|
|
1360
|
+
backlog remain separate. Reading status never rechunks or changes the policy.
|
|
1361
|
+
Sync results may include `rechunkedMirrors` when a policy change updates cached
|
|
1362
|
+
layouts. A stale client must reopen on `CHUNKING_POLICY_CONFLICT`.
|
|
1363
|
+
|
|
1355
1364
|
---
|
|
1356
1365
|
|
|
1357
1366
|
### gno_recall
|
|
@@ -24,6 +24,47 @@
|
|
|
24
24
|
"resident"
|
|
25
25
|
],
|
|
26
26
|
"properties": {
|
|
27
|
+
"chunking": {
|
|
28
|
+
"type": "object",
|
|
29
|
+
"additionalProperties": false,
|
|
30
|
+
"required": [
|
|
31
|
+
"configured",
|
|
32
|
+
"applied",
|
|
33
|
+
"state",
|
|
34
|
+
"pendingDocuments",
|
|
35
|
+
"pendingMirrors"
|
|
36
|
+
],
|
|
37
|
+
"properties": {
|
|
38
|
+
"configured": { "$ref": "#/properties/chunking/definitions/params" },
|
|
39
|
+
"applied": {
|
|
40
|
+
"oneOf": [
|
|
41
|
+
{ "$ref": "#/properties/chunking/definitions/params" },
|
|
42
|
+
{ "type": "null" }
|
|
43
|
+
]
|
|
44
|
+
},
|
|
45
|
+
"state": {
|
|
46
|
+
"type": "string",
|
|
47
|
+
"enum": ["empty", "legacy-default", "current", "pending", "mixed"]
|
|
48
|
+
},
|
|
49
|
+
"pendingDocuments": { "type": "integer", "minimum": 0 },
|
|
50
|
+
"pendingMirrors": { "type": "integer", "minimum": 0 }
|
|
51
|
+
},
|
|
52
|
+
"definitions": {
|
|
53
|
+
"params": {
|
|
54
|
+
"type": "object",
|
|
55
|
+
"additionalProperties": false,
|
|
56
|
+
"required": ["maxTokens", "overlapPercent"],
|
|
57
|
+
"properties": {
|
|
58
|
+
"maxTokens": {
|
|
59
|
+
"type": "integer",
|
|
60
|
+
"minimum": 10,
|
|
61
|
+
"maximum": 2251799813685247
|
|
62
|
+
},
|
|
63
|
+
"overlapPercent": { "type": "number", "minimum": 0, "maximum": 0.5 }
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
},
|
|
27
68
|
"resident": {
|
|
28
69
|
"$ref": "gno://schemas/resident-status@1.0"
|
|
29
70
|
},
|
|
@@ -160,6 +160,12 @@ export function formatSyncResultLines(
|
|
|
160
160
|
): string[] {
|
|
161
161
|
const lines: string[] = [];
|
|
162
162
|
|
|
163
|
+
if (syncResult.rechunkedMirrors) {
|
|
164
|
+
lines.push(
|
|
165
|
+
`Rechunked ${syncResult.rechunkedMirrors} cached mirrors. Run gno embed if embedding was skipped.`
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
|
|
163
169
|
for (const c of syncResult.collections) {
|
|
164
170
|
lines.push(`${c.collection}:`);
|
|
165
171
|
lines.push(
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
} from "../../config";
|
|
20
20
|
import { isConnectorActivationComplete } from "../../core/activation-connector-health";
|
|
21
21
|
import { buildActivationStatus } from "../../core/activation-status";
|
|
22
|
+
import { formatChunkingStatus } from "../../core/chunking-status";
|
|
22
23
|
import {
|
|
23
24
|
buildMemoryStatus,
|
|
24
25
|
formatMemoryStatusLines,
|
|
@@ -112,6 +113,9 @@ function formatTerminal(
|
|
|
112
113
|
lines.push(`Embedding backlog: ${indexStatus.embeddingBacklog} chunks`);
|
|
113
114
|
}
|
|
114
115
|
|
|
116
|
+
const chunking = formatChunkingStatus(indexStatus.chunking);
|
|
117
|
+
if (chunking) lines.push(chunking);
|
|
118
|
+
|
|
115
119
|
if (indexStatus.recentErrors > 0) {
|
|
116
120
|
lines.push(`Recent errors: ${indexStatus.recentErrors} (last 24h)`);
|
|
117
121
|
}
|
|
@@ -198,6 +202,9 @@ function formatMarkdown(
|
|
|
198
202
|
lines.push(`- **Embedding backlog**: ${indexStatus.embeddingBacklog}`);
|
|
199
203
|
lines.push(`- **Recent errors**: ${indexStatus.recentErrors}`);
|
|
200
204
|
|
|
205
|
+
const chunking = formatChunkingStatus(indexStatus.chunking);
|
|
206
|
+
if (chunking) lines.push(`- ${chunking}`);
|
|
207
|
+
|
|
201
208
|
if (indexStatus.lastUpdatedAt) {
|
|
202
209
|
lines.push(`- **Last updated**: ${indexStatus.lastUpdatedAt}`);
|
|
203
210
|
}
|
|
@@ -274,6 +281,7 @@ export async function status(
|
|
|
274
281
|
try {
|
|
275
282
|
const statusResult = await store.getStatus({
|
|
276
283
|
embedModel: resolveModelUri(config, "embed"),
|
|
284
|
+
chunking: config.chunking ?? {},
|
|
277
285
|
});
|
|
278
286
|
if (!statusResult.ok) {
|
|
279
287
|
return { success: false, error: statusResult.error.message };
|
|
@@ -339,6 +347,7 @@ export function formatStatus(
|
|
|
339
347
|
totalDocuments: s.activeDocuments,
|
|
340
348
|
totalChunks: s.totalChunks,
|
|
341
349
|
embeddingBacklog: s.embeddingBacklog,
|
|
350
|
+
chunking: s.chunking,
|
|
342
351
|
lastUpdated: s.lastUpdatedAt,
|
|
343
352
|
healthy: isStatusHealthy(s, result.activation),
|
|
344
353
|
contentTypeBoost: result.contentTypeBoost,
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/** Existing character-based chunker defaults. */
|
|
4
|
+
export const DEFAULT_CHUNKING_PARAMS = {
|
|
5
|
+
maxTokens: 800,
|
|
6
|
+
overlapPercent: 0.15,
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export const MAX_CHUNK_TOKENS = Math.floor(Number.MAX_SAFE_INTEGER / 4);
|
|
10
|
+
|
|
11
|
+
export const ChunkingConfigSchema = z
|
|
12
|
+
.object({
|
|
13
|
+
maxTokens: z
|
|
14
|
+
.number()
|
|
15
|
+
.int()
|
|
16
|
+
.min(10)
|
|
17
|
+
.max(MAX_CHUNK_TOKENS)
|
|
18
|
+
.default(DEFAULT_CHUNKING_PARAMS.maxTokens),
|
|
19
|
+
overlapPercent: z
|
|
20
|
+
.number()
|
|
21
|
+
.finite()
|
|
22
|
+
.min(0)
|
|
23
|
+
.max(0.5)
|
|
24
|
+
.default(DEFAULT_CHUNKING_PARAMS.overlapPercent),
|
|
25
|
+
})
|
|
26
|
+
.strict();
|
|
27
|
+
|
|
28
|
+
export type ChunkingParams = z.infer<typeof ChunkingConfigSchema>;
|
|
29
|
+
|
|
30
|
+
export function resolveChunkingParams(
|
|
31
|
+
input?: Partial<ChunkingParams>
|
|
32
|
+
): ChunkingParams {
|
|
33
|
+
return ChunkingConfigSchema.parse(input ?? {});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Canonical JSON is the policy identity, including partial/default aliases. */
|
|
37
|
+
export function chunkingPolicyKey(params: ChunkingParams): string {
|
|
38
|
+
return JSON.stringify({
|
|
39
|
+
maxTokens: params.maxTokens,
|
|
40
|
+
overlapPercent: params.overlapPercent,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const DEFAULT_CHUNKING_POLICY_KEY = chunkingPolicyKey(
|
|
45
|
+
DEFAULT_CHUNKING_PARAMS
|
|
46
|
+
);
|
package/src/config/types.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { z } from "zod";
|
|
|
12
12
|
import { URI_PREFIX } from "../app/constants";
|
|
13
13
|
import { JsonlFieldMappingSchema } from "../converters/adapters/jsonl/config";
|
|
14
14
|
import { MCP_TOOL_PROFILES } from "../mcp/tool-profile";
|
|
15
|
+
import { ChunkingConfigSchema, type ChunkingParams } from "./chunking";
|
|
15
16
|
import { RetrievalTraceConfigSchema } from "./retrieval-traces";
|
|
16
17
|
|
|
17
18
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -595,6 +596,9 @@ export const ConfigSchema = z.object({
|
|
|
595
596
|
/** Opt-in schema-lite content type rules */
|
|
596
597
|
contentTypes: z.array(ContentTypeSchema).default([]),
|
|
597
598
|
|
|
599
|
+
/** Optional index-wide chunk size/overlap; omitted preserves legacy defaults. */
|
|
600
|
+
chunking: ChunkingConfigSchema.optional(),
|
|
601
|
+
|
|
598
602
|
/** Model configuration */
|
|
599
603
|
models: ModelConfigSchema.optional(),
|
|
600
604
|
|
|
@@ -632,9 +636,10 @@ export const ConfigSchema = z.object({
|
|
|
632
636
|
|
|
633
637
|
export type Config = Omit<
|
|
634
638
|
z.infer<typeof ConfigSchema>,
|
|
635
|
-
"contentTypes" | "busyTimeoutMs"
|
|
639
|
+
"contentTypes" | "busyTimeoutMs" | "chunking"
|
|
636
640
|
> & {
|
|
637
641
|
contentTypes?: ContentTypeConfig[];
|
|
642
|
+
chunking?: Partial<ChunkingParams>;
|
|
638
643
|
/** Present after schema parse; omitted on hand-built Config objects. */
|
|
639
644
|
busyTimeoutMs?: number;
|
|
640
645
|
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ChunkingParams } from "../config/chunking";
|
|
2
|
+
import type { ChunkingStatus } from "../store/chunking";
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
chunkingPolicyKey,
|
|
6
|
+
DEFAULT_CHUNKING_POLICY_KEY,
|
|
7
|
+
} from "../config/chunking";
|
|
8
|
+
|
|
9
|
+
const formatParams = (params: ChunkingParams): string =>
|
|
10
|
+
`${params.maxTokens} approximate tokens / ${Number((params.overlapPercent * 100).toFixed(4))}% overlap`;
|
|
11
|
+
|
|
12
|
+
/** Keep the healthy default terminal output unchanged. */
|
|
13
|
+
export function formatChunkingStatus(status?: ChunkingStatus): string | null {
|
|
14
|
+
if (!status) return null;
|
|
15
|
+
if (
|
|
16
|
+
chunkingPolicyKey(status.configured) === DEFAULT_CHUNKING_POLICY_KEY &&
|
|
17
|
+
status.pendingMirrors === 0 &&
|
|
18
|
+
status.state !== "mixed"
|
|
19
|
+
)
|
|
20
|
+
return null;
|
|
21
|
+
const applied = status.applied
|
|
22
|
+
? formatParams(status.applied)
|
|
23
|
+
: status.state === "empty"
|
|
24
|
+
? "none"
|
|
25
|
+
: "mixed";
|
|
26
|
+
const pending = status.pendingMirrors
|
|
27
|
+
? `; ${status.pendingDocuments} documents / ${status.pendingMirrors} mirrors pending (run gno update)`
|
|
28
|
+
: "";
|
|
29
|
+
return `Chunking: configured ${formatParams(status.configured)}; applied ${applied}${pending}`;
|
|
30
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/** Shared mutation detection for resident content and vector generations. */
|
|
2
2
|
|
|
3
3
|
interface SyncMutationCounts {
|
|
4
|
+
rechunkedMirrors?: number;
|
|
4
5
|
filesAdded?: number;
|
|
5
6
|
filesUpdated?: number;
|
|
6
7
|
filesMarkedInactive?: number;
|
|
@@ -11,6 +12,7 @@ interface SyncMutationCounts {
|
|
|
11
12
|
|
|
12
13
|
export function hasContentMutation(result: SyncMutationCounts): boolean {
|
|
13
14
|
return (
|
|
15
|
+
(result.rechunkedMirrors ?? 0) > 0 ||
|
|
14
16
|
(result.filesAdded ?? result.totalFilesAdded ?? 0) > 0 ||
|
|
15
17
|
(result.filesUpdated ?? result.totalFilesUpdated ?? 0) > 0 ||
|
|
16
18
|
(result.filesMarkedInactive ?? 0) > 0 ||
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type { ChunkingPolicyToken } from "../store/chunking";
|
|
2
|
+
import type { ChunkInput, StorePort, StoreResult } from "../store/types";
|
|
3
|
+
import type { ChunkerPort, SyncOptions } from "./types";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
chunkingPolicyKey,
|
|
7
|
+
DEFAULT_CHUNKING_POLICY_KEY,
|
|
8
|
+
resolveChunkingParams,
|
|
9
|
+
} from "../config/chunking";
|
|
10
|
+
import { ChunkingPolicyConflictError } from "../store/chunking";
|
|
11
|
+
|
|
12
|
+
function unwrapChunking<T>(result: StoreResult<T>): T {
|
|
13
|
+
if (result.ok) return result.value;
|
|
14
|
+
if (result.error.code === "CHUNKING_POLICY_CONFLICT") {
|
|
15
|
+
throw new ChunkingPolicyConflictError();
|
|
16
|
+
}
|
|
17
|
+
throw new Error(`Chunking failed: ${result.error.message}`);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** One preparation per sync, including nested collection/path syncs. */
|
|
21
|
+
export async function prepareChunking(
|
|
22
|
+
store: StorePort,
|
|
23
|
+
chunker: ChunkerPort,
|
|
24
|
+
options: SyncOptions
|
|
25
|
+
): Promise<{ options: SyncOptions; rechunkedMirrors: number }> {
|
|
26
|
+
if (options.chunkingToken) return { options, rechunkedMirrors: 0 };
|
|
27
|
+
const params = resolveChunkingParams(options.chunking);
|
|
28
|
+
if (
|
|
29
|
+
!(
|
|
30
|
+
store.claimChunkingPolicy &&
|
|
31
|
+
store.listPendingChunkingMirrors &&
|
|
32
|
+
store.applyChunkLayout
|
|
33
|
+
)
|
|
34
|
+
) {
|
|
35
|
+
if (chunkingPolicyKey(params) !== DEFAULT_CHUNKING_POLICY_KEY) {
|
|
36
|
+
throw new Error("This store does not support configurable chunking");
|
|
37
|
+
}
|
|
38
|
+
return { options, rechunkedMirrors: 0 };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const token = unwrapChunking(await store.claimChunkingPolicy(params));
|
|
42
|
+
let rechunkedMirrors = 0;
|
|
43
|
+
let afterHash = "";
|
|
44
|
+
for (;;) {
|
|
45
|
+
const mirrors = unwrapChunking(
|
|
46
|
+
await store.listPendingChunkingMirrors(token, afterHash)
|
|
47
|
+
);
|
|
48
|
+
if (mirrors.length === 0) break;
|
|
49
|
+
for (const mirror of mirrors) {
|
|
50
|
+
const markdown = unwrapChunking(
|
|
51
|
+
await store.getContent(mirror.mirrorHash)
|
|
52
|
+
);
|
|
53
|
+
if (markdown === null)
|
|
54
|
+
throw new Error("Cached mirror disappeared during rechunking");
|
|
55
|
+
const chunks: ChunkInput[] = chunker
|
|
56
|
+
.chunk(markdown, token.params, mirror.languageHint, mirror.sourcePath)
|
|
57
|
+
.map((chunk) => ({
|
|
58
|
+
seq: chunk.seq,
|
|
59
|
+
pos: chunk.pos,
|
|
60
|
+
text: chunk.text,
|
|
61
|
+
startLine: chunk.startLine,
|
|
62
|
+
endLine: chunk.endLine,
|
|
63
|
+
language: chunk.language ?? undefined,
|
|
64
|
+
tokenCount: chunk.tokenCount ?? undefined,
|
|
65
|
+
}));
|
|
66
|
+
unwrapChunking(
|
|
67
|
+
await store.applyChunkLayout(
|
|
68
|
+
mirror.mirrorHash,
|
|
69
|
+
chunks,
|
|
70
|
+
token,
|
|
71
|
+
mirror.sourcePath,
|
|
72
|
+
mirror.languageHint
|
|
73
|
+
)
|
|
74
|
+
);
|
|
75
|
+
rechunkedMirrors += 1;
|
|
76
|
+
afterHash = mirror.mirrorHash;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
options: { ...options, chunking: token.params, chunkingToken: token },
|
|
81
|
+
rechunkedMirrors,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Preserve the original store contract for default-only test/alternate ports. */
|
|
86
|
+
export async function persistChunkLayout(
|
|
87
|
+
store: StorePort,
|
|
88
|
+
mirrorHash: string,
|
|
89
|
+
chunks: ChunkInput[],
|
|
90
|
+
token: ChunkingPolicyToken | undefined,
|
|
91
|
+
sourcePath: string,
|
|
92
|
+
languageHint?: string
|
|
93
|
+
): Promise<void> {
|
|
94
|
+
if (token && store.applyChunkLayout) {
|
|
95
|
+
unwrapChunking(
|
|
96
|
+
await store.applyChunkLayout(
|
|
97
|
+
mirrorHash,
|
|
98
|
+
chunks,
|
|
99
|
+
token,
|
|
100
|
+
sourcePath,
|
|
101
|
+
languageHint
|
|
102
|
+
)
|
|
103
|
+
);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
unwrapChunking(await store.upsertChunks(mirrorHash, chunks));
|
|
107
|
+
unwrapChunking(await store.rebuildFtsForHash(mirrorHash));
|
|
108
|
+
}
|