@gmickel/gno 2.2.1 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -117,7 +117,7 @@ gno daemon --detach # headless indexing + resident MCP gateway
117
117
 
118
118
  <!-- public-truth:current-version -->
119
119
 
120
- > Current source version: **v2.2.1**. See [CHANGELOG.md](./CHANGELOG.md).
120
+ > Current source version: **v2.3.1**. See [CHANGELOG.md](./CHANGELOG.md).
121
121
 
122
122
  <!-- /public-truth -->
123
123
 
@@ -288,6 +288,9 @@ gno query "ECONNREFUSED 127.0.0.1:5432" --thorough
288
288
 
289
289
  ```bash
290
290
  gno setup ~/notes --name notes # Build BM25 and prove an exact local result
291
+ gno skill install --target claude --scope user # Choose your agent
292
+ gno agents install --target claude # Teach retrieval and writing discipline
293
+ gno agents verify --target claude
291
294
  gno daemon --detach # Keep index fresh in the background (macOS/Linux)
292
295
  gno query "auth best practices" # Hybrid search
293
296
  gno ask "summarize the API" --answer # AI answer with citations
@@ -365,6 +368,30 @@ See also: [docs/DAEMON.md](./docs/DAEMON.md)
365
368
 
366
369
  ### Connect to AI Agents
367
370
 
371
+ For a second brain or LLM wiki, install both a connector and the retrieval
372
+ protocol. The skill teaches GNO commands and workflows; the instruction block
373
+ teaches the harness when to retrieve, how to choose an efficient search path,
374
+ and how to cite and maintain knowledge. A connector alone does not establish
375
+ that discipline.
376
+
377
+ ```bash
378
+ gno skill install --target claude --scope user
379
+ gno agents install --target claude
380
+ gno agents verify --target claude
381
+ ```
382
+
383
+ Choose your harness target (`claude`, `codex`, `opencode`, `openclaw`, or
384
+ `hermes`). For MCP clients, install the MCP connector instead of the skill,
385
+ then add the protocol where the harness supports it. `gno agents install`
386
+ without a target covers detected supported harnesses; see the
387
+ [agent instructions guide](docs/AGENT-INSTRUCTIONS.md) for the target matrix and
388
+ manual guidance. It preserves text outside its managed block.
389
+
390
+ Start a fresh agent session and ask a question answered by an indexed document.
391
+ Check that the agent retrieves it and cites the source. `gno agents verify`
392
+ checks the installed block, not model behavior. CLI and Web UI users can skip
393
+ agent setup.
394
+
368
395
  #### MCP Server (Claude Desktop, Cursor, Zed, etc.)
369
396
 
370
397
  One command to add GNO to your AI assistant:
@@ -391,7 +418,7 @@ target, add `--force` to preview the replacement without writing it.
391
418
 
392
419
  Check status: `gno mcp status`
393
420
 
394
- #### Skills (Claude Code, Codex, OpenCode, OpenClaw)
421
+ #### Skills (Claude Code, Codex, OpenCode, OpenClaw, Hermes)
395
422
 
396
423
  Skills integrate via CLI with no MCP overhead and include second-brain recipe playbooks:
397
424
 
@@ -400,6 +427,7 @@ gno skill install --scope user # User-wide
400
427
  gno skill install --target codex # Codex
401
428
  gno skill install --target opencode # OpenCode
402
429
  gno skill install --target openclaw # OpenClaw
430
+ gno skill install --target hermes # Hermes
403
431
  gno skill install --target all # All targets
404
432
  ```
405
433
 
@@ -627,10 +655,12 @@ Give your local LLM agents a long-term memory. GNO integrates as a Claude Code s
627
655
 
628
656
  ### Skills
629
657
 
630
- Skills add GNO search to Claude Code, Codex, OpenCode, and OpenClaw without MCP protocol overhead:
658
+ Skills teach Claude Code, Codex, OpenCode, OpenClaw, and Hermes how to use GNO. Pair the skill with the retrieval protocol for second-brain and LLM-wiki work:
631
659
 
632
660
  ```bash
633
661
  gno skill install --scope user
662
+ gno agents install --target claude
663
+ gno agents verify --target claude
634
664
  ```
635
665
 
636
666
  ![GNO Skill in Claude Code](./assets/screenshots/claudecodeskill.jpg)
@@ -818,7 +848,23 @@ On [Omarchy](https://omarchy.org/), [**GNO Recall**](https://github.com/gmickel/
818
848
  omarchy plugin add https://github.com/gmickel/omarchy-gno-recall --enable
819
849
  ```
820
850
 
821
- Requires gno >= 1.36.0 on `PATH`.
851
+ Then install the plugin's verified runtime and, optionally, the Super+R binding:
852
+
853
+ ```bash
854
+ cd ~/.config/omarchy/plugins/gmickel.gno-recall
855
+ ./scripts/install-runtime.sh
856
+ ./scripts/install-keybind.sh # Optional; checks for conflicts
857
+ ```
858
+
859
+ [Marketplace listing](https://omarchyplugins.com/plugin.html?id=gmickel.gno-recall)
860
+ · [Setup and runtime compatibility](https://github.com/gmickel/omarchy-gno-recall#install)
861
+ · [Screenshots](https://github.com/gmickel/omarchy-gno-recall#screenshots)
862
+
863
+ Recall uses its own verified GNO/Bun runtime. A global GNO upgrade does not
864
+ update it; follow the plugin's update instructions and shared-index
865
+ compatibility guidance.
866
+
867
+ ![GNO Recall searching a demo index](https://raw.githubusercontent.com/gmickel/omarchy-gno-recall/main/assets/screenshots/overlay-search.png)
822
868
 
823
869
  ---
824
870
 
@@ -1,10 +1,10 @@
1
1
  # GNO — Agent Skill
2
2
 
3
3
  Local-first semantic search for documents, notes, and knowledge bases,
4
- packaged as a Claude Code / Codex / OpenCode / OpenClaw skill.
4
+ packaged as a Claude Code / Codex / OpenCode / OpenClaw / Hermes skill.
5
5
 
6
6
  > **TL;DR** — this folder is a runnable agent skill. Drop it into any
7
- > Claude Code, Codex, OpenCode, or OpenClaw workspace and the agent can
7
+ > Claude Code, Codex, OpenCode, OpenClaw, or Hermes workspace and the agent can
8
8
  > index and search your local files through the `gno` CLI.
9
9
 
10
10
  ## Prerequisites
@@ -13,14 +13,10 @@ You need the `gno` CLI installed locally. The skill drives it; it does
13
13
  not ship the binary itself.
14
14
 
15
15
  ```bash
16
- # macOS / Linux
17
- curl -fsSL https://gno.sh/install | bash
18
-
19
- # npm / Bun
20
- bun add -g @gmickel/gno # or: npm install -g @gmickel/gno
16
+ bun install -g @gmickel/gno
21
17
  ```
22
18
 
23
- Verify: `gno --version` should print `1.0.3` or later.
19
+ Verify the installed release with `gno --version`. Use the bundled skill to match that release.
24
20
 
25
21
  ## Install the skill
26
22
 
@@ -30,8 +26,7 @@ you're starting from.
30
26
  ### Option A — install from your local `gno` (recommended if you already have GNO)
31
27
 
32
28
  If you already installed GNO, it ships this skill in-tree. One command
33
- drops it into the right place for every supported agent, always in sync
34
- with your installed GNO version:
29
+ drops it into the right place for every supported agent, matching the GNO version installed when you run it:
35
30
 
36
31
  ```bash
37
32
  gno skill install --target claude # Claude Code (default)
@@ -59,23 +54,67 @@ gno skill paths # resolved installation paths
59
54
  gno skill uninstall --target all # remove from all agents
60
55
  ```
61
56
 
62
- This path always gives you the skill that matches your GNO CLI — no
63
- version drift.
57
+ After upgrading GNO, refresh the installed copy as described below.
58
+
59
+ ### Option B: ClawHub-managed installation
60
+
61
+ Use the [GNO listing](https://clawhub.ai/gmickel/gno) when your harness manages
62
+ skills through a registry. The GNO executable is still required separately.
63
+
64
+ ```bash
65
+ # OpenClaw workspace: run from its workspace root
66
+ bunx clawhub@0.23.3 install @gmickel/gno
67
+ bunx clawhub@0.23.3 update @gmickel/gno
68
+
69
+ # Hermes hub installation
70
+ hermes skills install clawhub/gno # May require review; prefer the local installer
71
+ hermes skills check
72
+ hermes skills update gno
73
+ ```
74
+
75
+ GNO releases publish the same source bundle to ClawHub. Choose one installer
76
+ for a given skill directory so registry and local copies do not overwrite one
77
+ another. For Hermes, prefer the local GNO installer if your hub version cannot
78
+ retrieve the full bundle. A bare SKILL.md URL may omit recipes and references.
64
79
 
65
- ### Option B — install from ClawHub (recommended if you use OpenClaw)
80
+ ## Install the retrieval protocol too
66
81
 
67
- ClawHub is the OpenClaw skill registry. Use this when you manage
68
- skills centrally across multiple workspaces or when you don't have GNO
69
- installed yet.
82
+ For second-brain and LLM-wiki work, pair the skill with standing instructions
83
+ for the harness. The skill teaches commands and workflows; the protocol teaches
84
+ when to retrieve, which search path to use, and how to cite and maintain knowledge.
70
85
 
71
86
  ```bash
72
- openclaw skills install gno # installs @gmickel/gno
73
- openclaw skills update gno # pull a newer version later
74
- openclaw skills info gno # inspect what's installed
87
+ gno agents install # Detected supported harnesses
88
+ gno agents verify
75
89
  ```
76
90
 
77
- You can also browse the skill at
78
- <https://clawhub.ai/gmickel/gno> and download the folder manually.
91
+ Use `--target claude`, `codex`, `opencode`, `openclaw`, or `hermes` for one
92
+ harness. MCP users should install the protocol too where supported. Text outside
93
+ the managed instruction block stays unchanged. Start a fresh session and verify
94
+ that the agent retrieves and cites a document from your index. The verify command
95
+ checks the block, not model behavior. See [agent instructions](https://gno.sh/docs/agents-install).
96
+
97
+ ## Keep installations current
98
+
99
+ Local installs copy the skill from your installed GNO release. After upgrading,
100
+ refresh the selected target and protocol:
101
+
102
+ ```bash
103
+ gno skill install --target hermes --scope user --force
104
+ gno agents update --target hermes
105
+ gno agents verify --target hermes
106
+ ```
107
+
108
+ `--force` replaces local skill edits. Preserve your customizations before using
109
+ it. Change the target for another harness. Registry installs use their native
110
+ update command instead; upgrading the GNO executable alone does not refresh them.
111
+
112
+ For unattended registry updates, schedule only the chosen skill's native update
113
+ command with your OS scheduler, using the same user, workspace, and executable
114
+ paths as the interactive install. Preserve logs and keep safety prompts enabled;
115
+ an update that needs approval must remain pending. Run `gno agents update` after
116
+ upgrading GNO. New skill versions can require a newer executable, so update GNO
117
+ before refreshing the skill. Publishing a release does not modify users' machines.
79
118
 
80
119
  ## What the skill teaches the agent
81
120
 
@@ -509,6 +509,13 @@ gno embed --collection travel
509
509
 
510
510
  MCP `gno.sync` and `gno.capture` do NOT auto-embed. Use CLI for embedding.
511
511
 
512
+ Optional index-wide YAML `chunking.maxTokens` and `chunking.overlapPercent`
513
+ control size and overlap (`0.15` means 15%). Leave defaults unless asked to
514
+ tune them. After a change, `gno index` rechunks cached mirrors and embeds;
515
+ `gno update` rechunks only. Check `gno status --json` fields
516
+ `chunking.pendingMirrors` and `embeddingBacklog`. Use separate `--config`
517
+ files and `--index` names for comparisons; see [cli-reference.md](cli-reference.md).
518
+
512
519
  ## Capture Notes
513
520
 
514
521
  Use `gno capture` for quick second-brain writes into an editable collection:
@@ -24,6 +24,22 @@ punctuation. NFC/case-equivalent spellings share one identity. See
24
24
 
25
25
  ## Initialization
26
26
 
27
+ ### Optional index-wide chunking
28
+
29
+ User YAML can set `chunking: { maxTokens: 256, overlapPercent: 0.15 }`.
30
+ Defaults remain 800 approximate tokens and 0.15 overlap; unchanged/default
31
+ config does not force a rebuild. The token estimate uses four characters per
32
+ token, and overlap accepts a fraction from 0 through 0.5.
33
+
34
+ After changing policy, `gno index` rechunks cached mirrors and embeds changed
35
+ chunks. `gno update` performs the rechunking without embedding. The policy
36
+ applies across the index even for targeted source sync; original files are not
37
+ read merely to rechunk. `gno status --json` reports configured/applied policy,
38
+ pending mirrors/documents, and a separate embedding backlog. On
39
+ `CHUNKING_POLICY_CONFLICT`, reopen the client with the intended configuration.
40
+ Compare policies with separate `--config` files and `--index` names, holding
41
+ corpus, models, query set, and type boosts constant.
42
+
27
43
  ### gno setup
28
44
 
29
45
  Preferred activation path: add one folder, prove a real exact lexical result,
@@ -0,0 +1 @@
1
+ 674db0a56d6ba5a485f4d6195066a7cd700b3dbbbcbf921b12047548fcdd21e0 gno-browser-clipper-v2.3.1.zip
@@ -21,5 +21,5 @@
21
21
  "content_security_policy": {
22
22
  "extension_pages": "script-src 'self'; object-src 'none'; connect-src http://127.0.0.1:*"
23
23
  },
24
- "version": "2.2.1"
24
+ "version": "2.3.1"
25
25
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gmickel/gno",
3
- "version": "2.2.1",
3
+ "version": "2.3.1",
4
4
  "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
5
5
  "keywords": [
6
6
  "embeddings",
package/spec/cli.md CHANGED
@@ -531,6 +531,27 @@ protection remain authoritative.
531
531
  `graphHints` is active: ordered hints type
532
532
  projected wiki/markdown edges and surface in graph traversal/diagnose metadata.
533
533
 
534
+ Optional root `chunking` sets one policy per index. `maxTokens` defaults to 800
535
+ and accepts integers from 10 through 2251799813685247; the chunker estimates
536
+ four characters per token. `overlapPercent` defaults to 0.15 and accepts a
537
+ fraction from 0 through 0.5. Missing fields inherit defaults. Invalid values
538
+ fail configuration validation before index mutation.
539
+
540
+ Omitted or explicitly default settings preserve existing chunks and embeddings
541
+ on upgrade. A policy change rechunks cached Markdown mirrors during the next
542
+ index/update or targeted sync, including mirrors outside the source-refresh
543
+ collection. Rechunking does not read original files or claim source freshness.
544
+ Sync-only operations leave changed chunks pending embedding; `gno index`
545
+ also embeds. A stale writer receives `CHUNKING_POLICY_CONFLICT` and must reopen
546
+ its client/runtime with the intended configuration.
547
+
548
+ Status JSON adds `chunking` with `configured` parameters, `applied` parameters
549
+ (null for empty/mixed layouts), `state` (`empty`, `legacy-default`, `current`,
550
+ `pending`, or `mixed`), `pendingDocuments`, and `pendingMirrors`. These counts
551
+ cover active cached content, independently of source errors and embedding
552
+ backlog. Sync receipts optionally add `rechunkedMirrors` when cached layouts
553
+ were updated; existing file counters keep their source-refresh meaning.
554
+
534
555
  `collections[].sourceAvailability` is optional; omitted means `any`. Exact
535
556
  values: `any` | `local`. There is no separate public knob beyond these two
536
557
  modes and no claim that availability is egress policy. `any` preserves
package/spec/mcp.md CHANGED
@@ -1352,6 +1352,15 @@ counters; it never claims attachment to another process.
1352
1352
  normalized IDs/factors plus the rules fingerprint; path prefixes are never
1353
1353
  returned.
1354
1354
 
1355
+ `chunking` reports the index-wide configuration and cached layout readiness:
1356
+ `configured` and `applied` contain `maxTokens` and `overlapPercent` (`applied`
1357
+ is null for empty/mixed layouts); `state` is `empty`, `legacy-default`,
1358
+ `current`, `pending`, or `mixed`; `pendingDocuments` and `pendingMirrors` count
1359
+ active cached content using another policy. Source refresh errors and embedding
1360
+ backlog remain separate. Reading status never rechunks or changes the policy.
1361
+ Sync results may include `rechunkedMirrors` when a policy change updates cached
1362
+ layouts. A stale client must reopen on `CHUNKING_POLICY_CONFLICT`.
1363
+
1355
1364
  ---
1356
1365
 
1357
1366
  ### gno_recall
@@ -24,6 +24,47 @@
24
24
  "resident"
25
25
  ],
26
26
  "properties": {
27
+ "chunking": {
28
+ "type": "object",
29
+ "additionalProperties": false,
30
+ "required": [
31
+ "configured",
32
+ "applied",
33
+ "state",
34
+ "pendingDocuments",
35
+ "pendingMirrors"
36
+ ],
37
+ "properties": {
38
+ "configured": { "$ref": "#/properties/chunking/definitions/params" },
39
+ "applied": {
40
+ "oneOf": [
41
+ { "$ref": "#/properties/chunking/definitions/params" },
42
+ { "type": "null" }
43
+ ]
44
+ },
45
+ "state": {
46
+ "type": "string",
47
+ "enum": ["empty", "legacy-default", "current", "pending", "mixed"]
48
+ },
49
+ "pendingDocuments": { "type": "integer", "minimum": 0 },
50
+ "pendingMirrors": { "type": "integer", "minimum": 0 }
51
+ },
52
+ "definitions": {
53
+ "params": {
54
+ "type": "object",
55
+ "additionalProperties": false,
56
+ "required": ["maxTokens", "overlapPercent"],
57
+ "properties": {
58
+ "maxTokens": {
59
+ "type": "integer",
60
+ "minimum": 10,
61
+ "maximum": 2251799813685247
62
+ },
63
+ "overlapPercent": { "type": "number", "minimum": 0, "maximum": 0.5 }
64
+ }
65
+ }
66
+ }
67
+ },
27
68
  "resident": {
28
69
  "$ref": "gno://schemas/resident-status@1.0"
29
70
  },
@@ -160,6 +160,12 @@ export function formatSyncResultLines(
160
160
  ): string[] {
161
161
  const lines: string[] = [];
162
162
 
163
+ if (syncResult.rechunkedMirrors) {
164
+ lines.push(
165
+ `Rechunked ${syncResult.rechunkedMirrors} cached mirrors. Run gno embed if embedding was skipped.`
166
+ );
167
+ }
168
+
163
169
  for (const c of syncResult.collections) {
164
170
  lines.push(`${c.collection}:`);
165
171
  lines.push(
@@ -19,6 +19,7 @@ import {
19
19
  } from "../../config";
20
20
  import { isConnectorActivationComplete } from "../../core/activation-connector-health";
21
21
  import { buildActivationStatus } from "../../core/activation-status";
22
+ import { formatChunkingStatus } from "../../core/chunking-status";
22
23
  import {
23
24
  buildMemoryStatus,
24
25
  formatMemoryStatusLines,
@@ -112,6 +113,9 @@ function formatTerminal(
112
113
  lines.push(`Embedding backlog: ${indexStatus.embeddingBacklog} chunks`);
113
114
  }
114
115
 
116
+ const chunking = formatChunkingStatus(indexStatus.chunking);
117
+ if (chunking) lines.push(chunking);
118
+
115
119
  if (indexStatus.recentErrors > 0) {
116
120
  lines.push(`Recent errors: ${indexStatus.recentErrors} (last 24h)`);
117
121
  }
@@ -198,6 +202,9 @@ function formatMarkdown(
198
202
  lines.push(`- **Embedding backlog**: ${indexStatus.embeddingBacklog}`);
199
203
  lines.push(`- **Recent errors**: ${indexStatus.recentErrors}`);
200
204
 
205
+ const chunking = formatChunkingStatus(indexStatus.chunking);
206
+ if (chunking) lines.push(`- ${chunking}`);
207
+
201
208
  if (indexStatus.lastUpdatedAt) {
202
209
  lines.push(`- **Last updated**: ${indexStatus.lastUpdatedAt}`);
203
210
  }
@@ -274,6 +281,7 @@ export async function status(
274
281
  try {
275
282
  const statusResult = await store.getStatus({
276
283
  embedModel: resolveModelUri(config, "embed"),
284
+ chunking: config.chunking ?? {},
277
285
  });
278
286
  if (!statusResult.ok) {
279
287
  return { success: false, error: statusResult.error.message };
@@ -339,6 +347,7 @@ export function formatStatus(
339
347
  totalDocuments: s.activeDocuments,
340
348
  totalChunks: s.totalChunks,
341
349
  embeddingBacklog: s.embeddingBacklog,
350
+ chunking: s.chunking,
342
351
  lastUpdated: s.lastUpdatedAt,
343
352
  healthy: isStatusHealthy(s, result.activation),
344
353
  contentTypeBoost: result.contentTypeBoost,
@@ -0,0 +1,46 @@
1
+ import { z } from "zod";
2
+
3
+ /** Existing character-based chunker defaults. */
4
+ export const DEFAULT_CHUNKING_PARAMS = {
5
+ maxTokens: 800,
6
+ overlapPercent: 0.15,
7
+ };
8
+
9
+ export const MAX_CHUNK_TOKENS = Math.floor(Number.MAX_SAFE_INTEGER / 4);
10
+
11
+ export const ChunkingConfigSchema = z
12
+ .object({
13
+ maxTokens: z
14
+ .number()
15
+ .int()
16
+ .min(10)
17
+ .max(MAX_CHUNK_TOKENS)
18
+ .default(DEFAULT_CHUNKING_PARAMS.maxTokens),
19
+ overlapPercent: z
20
+ .number()
21
+ .finite()
22
+ .min(0)
23
+ .max(0.5)
24
+ .default(DEFAULT_CHUNKING_PARAMS.overlapPercent),
25
+ })
26
+ .strict();
27
+
28
+ export type ChunkingParams = z.infer<typeof ChunkingConfigSchema>;
29
+
30
+ export function resolveChunkingParams(
31
+ input?: Partial<ChunkingParams>
32
+ ): ChunkingParams {
33
+ return ChunkingConfigSchema.parse(input ?? {});
34
+ }
35
+
36
+ /** Canonical JSON is the policy identity, including partial/default aliases. */
37
+ export function chunkingPolicyKey(params: ChunkingParams): string {
38
+ return JSON.stringify({
39
+ maxTokens: params.maxTokens,
40
+ overlapPercent: params.overlapPercent,
41
+ });
42
+ }
43
+
44
+ export const DEFAULT_CHUNKING_POLICY_KEY = chunkingPolicyKey(
45
+ DEFAULT_CHUNKING_PARAMS
46
+ );
@@ -12,6 +12,7 @@ import { z } from "zod";
12
12
  import { URI_PREFIX } from "../app/constants";
13
13
  import { JsonlFieldMappingSchema } from "../converters/adapters/jsonl/config";
14
14
  import { MCP_TOOL_PROFILES } from "../mcp/tool-profile";
15
+ import { ChunkingConfigSchema, type ChunkingParams } from "./chunking";
15
16
  import { RetrievalTraceConfigSchema } from "./retrieval-traces";
16
17
 
17
18
  // ─────────────────────────────────────────────────────────────────────────────
@@ -595,6 +596,9 @@ export const ConfigSchema = z.object({
595
596
  /** Opt-in schema-lite content type rules */
596
597
  contentTypes: z.array(ContentTypeSchema).default([]),
597
598
 
599
+ /** Optional index-wide chunk size/overlap; omitted preserves legacy defaults. */
600
+ chunking: ChunkingConfigSchema.optional(),
601
+
598
602
  /** Model configuration */
599
603
  models: ModelConfigSchema.optional(),
600
604
 
@@ -632,9 +636,10 @@ export const ConfigSchema = z.object({
632
636
 
633
637
  export type Config = Omit<
634
638
  z.infer<typeof ConfigSchema>,
635
- "contentTypes" | "busyTimeoutMs"
639
+ "contentTypes" | "busyTimeoutMs" | "chunking"
636
640
  > & {
637
641
  contentTypes?: ContentTypeConfig[];
642
+ chunking?: Partial<ChunkingParams>;
638
643
  /** Present after schema parse; omitted on hand-built Config objects. */
639
644
  busyTimeoutMs?: number;
640
645
  };
@@ -0,0 +1,30 @@
1
+ import type { ChunkingParams } from "../config/chunking";
2
+ import type { ChunkingStatus } from "../store/chunking";
3
+
4
+ import {
5
+ chunkingPolicyKey,
6
+ DEFAULT_CHUNKING_POLICY_KEY,
7
+ } from "../config/chunking";
8
+
9
+ const formatParams = (params: ChunkingParams): string =>
10
+ `${params.maxTokens} approximate tokens / ${Number((params.overlapPercent * 100).toFixed(4))}% overlap`;
11
+
12
+ /** Keep the healthy default terminal output unchanged. */
13
+ export function formatChunkingStatus(status?: ChunkingStatus): string | null {
14
+ if (!status) return null;
15
+ if (
16
+ chunkingPolicyKey(status.configured) === DEFAULT_CHUNKING_POLICY_KEY &&
17
+ status.pendingMirrors === 0 &&
18
+ status.state !== "mixed"
19
+ )
20
+ return null;
21
+ const applied = status.applied
22
+ ? formatParams(status.applied)
23
+ : status.state === "empty"
24
+ ? "none"
25
+ : "mixed";
26
+ const pending = status.pendingMirrors
27
+ ? `; ${status.pendingDocuments} documents / ${status.pendingMirrors} mirrors pending (run gno update)`
28
+ : "";
29
+ return `Chunking: configured ${formatParams(status.configured)}; applied ${applied}${pending}`;
30
+ }
@@ -1,6 +1,7 @@
1
1
  /** Shared mutation detection for resident content and vector generations. */
2
2
 
3
3
  interface SyncMutationCounts {
4
+ rechunkedMirrors?: number;
4
5
  filesAdded?: number;
5
6
  filesUpdated?: number;
6
7
  filesMarkedInactive?: number;
@@ -11,6 +12,7 @@ interface SyncMutationCounts {
11
12
 
12
13
  export function hasContentMutation(result: SyncMutationCounts): boolean {
13
14
  return (
15
+ (result.rechunkedMirrors ?? 0) > 0 ||
14
16
  (result.filesAdded ?? result.totalFilesAdded ?? 0) > 0 ||
15
17
  (result.filesUpdated ?? result.totalFilesUpdated ?? 0) > 0 ||
16
18
  (result.filesMarkedInactive ?? 0) > 0 ||
@@ -0,0 +1,108 @@
1
+ import type { ChunkingPolicyToken } from "../store/chunking";
2
+ import type { ChunkInput, StorePort, StoreResult } from "../store/types";
3
+ import type { ChunkerPort, SyncOptions } from "./types";
4
+
5
+ import {
6
+ chunkingPolicyKey,
7
+ DEFAULT_CHUNKING_POLICY_KEY,
8
+ resolveChunkingParams,
9
+ } from "../config/chunking";
10
+ import { ChunkingPolicyConflictError } from "../store/chunking";
11
+
12
+ function unwrapChunking<T>(result: StoreResult<T>): T {
13
+ if (result.ok) return result.value;
14
+ if (result.error.code === "CHUNKING_POLICY_CONFLICT") {
15
+ throw new ChunkingPolicyConflictError();
16
+ }
17
+ throw new Error(`Chunking failed: ${result.error.message}`);
18
+ }
19
+
20
+ /** One preparation per sync, including nested collection/path syncs. */
21
+ export async function prepareChunking(
22
+ store: StorePort,
23
+ chunker: ChunkerPort,
24
+ options: SyncOptions
25
+ ): Promise<{ options: SyncOptions; rechunkedMirrors: number }> {
26
+ if (options.chunkingToken) return { options, rechunkedMirrors: 0 };
27
+ const params = resolveChunkingParams(options.chunking);
28
+ if (
29
+ !(
30
+ store.claimChunkingPolicy &&
31
+ store.listPendingChunkingMirrors &&
32
+ store.applyChunkLayout
33
+ )
34
+ ) {
35
+ if (chunkingPolicyKey(params) !== DEFAULT_CHUNKING_POLICY_KEY) {
36
+ throw new Error("This store does not support configurable chunking");
37
+ }
38
+ return { options, rechunkedMirrors: 0 };
39
+ }
40
+
41
+ const token = unwrapChunking(await store.claimChunkingPolicy(params));
42
+ let rechunkedMirrors = 0;
43
+ let afterHash = "";
44
+ for (;;) {
45
+ const mirrors = unwrapChunking(
46
+ await store.listPendingChunkingMirrors(token, afterHash)
47
+ );
48
+ if (mirrors.length === 0) break;
49
+ for (const mirror of mirrors) {
50
+ const markdown = unwrapChunking(
51
+ await store.getContent(mirror.mirrorHash)
52
+ );
53
+ if (markdown === null)
54
+ throw new Error("Cached mirror disappeared during rechunking");
55
+ const chunks: ChunkInput[] = chunker
56
+ .chunk(markdown, token.params, mirror.languageHint, mirror.sourcePath)
57
+ .map((chunk) => ({
58
+ seq: chunk.seq,
59
+ pos: chunk.pos,
60
+ text: chunk.text,
61
+ startLine: chunk.startLine,
62
+ endLine: chunk.endLine,
63
+ language: chunk.language ?? undefined,
64
+ tokenCount: chunk.tokenCount ?? undefined,
65
+ }));
66
+ unwrapChunking(
67
+ await store.applyChunkLayout(
68
+ mirror.mirrorHash,
69
+ chunks,
70
+ token,
71
+ mirror.sourcePath,
72
+ mirror.languageHint
73
+ )
74
+ );
75
+ rechunkedMirrors += 1;
76
+ afterHash = mirror.mirrorHash;
77
+ }
78
+ }
79
+ return {
80
+ options: { ...options, chunking: token.params, chunkingToken: token },
81
+ rechunkedMirrors,
82
+ };
83
+ }
84
+
85
+ /** Preserve the original store contract for default-only test/alternate ports. */
86
+ export async function persistChunkLayout(
87
+ store: StorePort,
88
+ mirrorHash: string,
89
+ chunks: ChunkInput[],
90
+ token: ChunkingPolicyToken | undefined,
91
+ sourcePath: string,
92
+ languageHint?: string
93
+ ): Promise<void> {
94
+ if (token && store.applyChunkLayout) {
95
+ unwrapChunking(
96
+ await store.applyChunkLayout(
97
+ mirrorHash,
98
+ chunks,
99
+ token,
100
+ sourcePath,
101
+ languageHint
102
+ )
103
+ );
104
+ return;
105
+ }
106
+ unwrapChunking(await store.upsertChunks(mirrorHash, chunks));
107
+ unwrapChunking(await store.rebuildFtsForHash(mirrorHash));
108
+ }