@gmickel/gno 2.2.1 → 2.3.1

package/src/ingestion/record-container.ts

@@ -23,6 +23,7 @@ import {
 } from "../core/change-diff";
 import { createEgressLineage } from "../core/egress-provenance";
 import { normalizeTag, validateTag } from "../core/tags";
+import { persistChunkLayout } from "./chunking";
 import { runRecordAdapter } from "./record-adapter";
 import { recordVirtualPath } from "./record-path";
 import { reconcileRecordSnapshot, type RecordSyncPlan } from "./record-sync";
@@ -397,7 +398,7 @@ const persistRecord = async (
     const chunks: ChunkInput[] = input.chunker
       .chunk(
         record.markdown,
-        DEFAULT_CHUNK_PARAMS,
+        input.options.chunkingToken?.params ?? DEFAULT_CHUNK_PARAMS,
         record.languageHint ?? input.collection.languageHint,
         virtualPath
       )
@@ -410,13 +411,13 @@ const persistRecord = async (
         language: chunk.language ?? undefined,
         tokenCount: chunk.tokenCount ?? undefined,
       }));
-    mustOk(
-      await input.store.upsertChunks(record.mirrorHash, chunks),
-      "upsertChunks"
-    );
-    mustOk(
-      await input.store.rebuildFtsForHash(record.mirrorHash),
-      "rebuildFtsForHash"
+    await persistChunkLayout(
+      input.store,
+      record.mirrorHash,
+      chunks,
+      input.options.chunkingToken,
+      virtualPath,
+      record.languageHint ?? input.collection.languageHint
     );
     mustOk(
       await input.store.setDocTags(document.id, categories, "frontmatter"),

package/src/ingestion/sync-options.ts

@@ -20,11 +20,13 @@ export function resolveContentTypeRules(
 
 export function withContentTypeRules(
   options: SyncOptions = {},
-  config?: Pick<Config, "contentTypes">
+  config?: Pick<Config, "contentTypes" | "chunking">
 ): SyncOptions {
   const rules = options.contentTypeRules ?? resolveContentTypeRules(config);
+  const chunking = options.chunking ?? config?.chunking;
   return {
     ...options,
+    ...(chunking ? { chunking } : {}),
     contentTypeRules: rules,
     contentTypeRulesFingerprint:
       options.contentTypeRulesFingerprint ??

package/src/ingestion/sync.ts

@@ -58,6 +58,7 @@ import {
 import { extractMemoryScopes } from "../core/memory-record";
 import { normalizeTag, validateTag } from "../core/tags";
 import { defaultChunker } from "./chunker";
+import { persistChunkLayout, prepareChunking } from "./chunking";
 import {
   extractHashtags,
   parseFrontmatter,
@@ -937,7 +938,7 @@ export class SyncService {
         // 9. Chunk content
         const chunks = this.chunker.chunk(
           artifact.markdown,
-          DEFAULT_CHUNK_PARAMS,
+          options.chunkingToken?.params ?? DEFAULT_CHUNK_PARAMS,
           artifact.languageHint ?? collection.languageHint,
           entry.relPath
         );
@@ -953,21 +954,15 @@ export class SyncService {
           tokenCount: c.tokenCount ?? undefined,
         }));
 
-        // 11. Upsert chunks - CHECKED
-        const chunksResult = await store.upsertChunks(
+        // 11-12. Apply chunks and lexical projection under the policy token.
+        await persistChunkLayout(
+          store,
           artifact.mirrorHash,
-          chunkInputs
+          chunkInputs,
+          options.chunkingToken,
+          entry.relPath,
+          artifact.languageHint ?? collection.languageHint
         );
-        mustOk(chunksResult, "upsertChunks", {
-          mirrorHash: artifact.mirrorHash,
-          chunkCount: chunkInputs.length,
-        });
-
-        // 12. Rebuild FTS for this hash - CHECKED
-        const ftsResult = await store.rebuildFtsForHash(artifact.mirrorHash);
-        mustOk(ftsResult, "rebuildFtsForHash", {
-          mirrorHash: artifact.mirrorHash,
-        });
 
         // 13. Extract and store tags from frontmatter and body hashtags
         // Always call setDocTags to clear removed tags on re-sync
@@ -1249,8 +1244,9 @@ export class SyncService {
     options: SyncOptions = {}
   ): Promise<CollectionSyncResult> {
     const startedAt = Date.now();
+    const prepared = await prepareChunking(store, this.chunker, options);
     const syncOptions: SyncOptions = {
-      ...options,
+      ...prepared.options,
       contentTypeRules: options.contentTypeRules ?? [],
       contentTypeRulesFingerprint:
         options.contentTypeRulesFingerprint ??
@@ -1511,13 +1507,18 @@ export class SyncService {
       syncOptions.projectTypedEdges === false
         ? []
         : await this.projectTypedEdges(store, syncOptions, projectionSourceIds);
-    return summarizePathResults(
-      collection.name,
-      results,
-      markedInactive,
-      startedAt,
-      errors
-    );
+    return {
+      ...summarizePathResults(
+        collection.name,
+        results,
+        markedInactive,
+        startedAt,
+        errors
+      ),
+      ...(prepared.rechunkedMirrors
+        ? { rechunkedMirrors: prepared.rechunkedMirrors }
+        : {}),
+    };
   }
 
   /**
@@ -1717,8 +1718,9 @@ export class SyncService {
     options: SyncOptions = {}
   ): Promise<CollectionSyncResult> {
     const startTime = Date.now();
+    const prepared = await prepareChunking(store, this.chunker, options);
     const syncOptions: SyncOptions = {
-      ...options,
+      ...prepared.options,
       contentTypeRules: options.contentTypeRules ?? [],
       contentTypeRulesFingerprint:
         options.contentTypeRulesFingerprint ??
@@ -2038,6 +2040,9 @@ export class SyncService {
 
     return {
       collection: collection.name,
+      ...(prepared.rechunkedMirrors
+        ? { rechunkedMirrors: prepared.rechunkedMirrors }
+        : {}),
       filesProcessed: entries.length + inventoryErrored,
       filesAdded: added,
       filesUpdated: updated,
@@ -2060,9 +2065,10 @@ export class SyncService {
     options: SyncOptions = {}
   ): Promise<SyncResult> {
     const startTime = Date.now();
+    const prepared = await prepareChunking(store, this.chunker, options);
     const results: CollectionSyncResult[] = [];
     const deferredProjectionOptions: SyncOptions = {
-      ...options,
+      ...prepared.options,
       projectTypedEdges: false,
     };
 
@@ -2094,6 +2100,9 @@ export class SyncService {
 
     return {
       collections: results,
+      ...(prepared.rechunkedMirrors
+        ? { rechunkedMirrors: prepared.rechunkedMirrors }
+        : {}),
       totalDurationMs: Date.now() - startTime,
       totalFilesProcessed: totals.processed,
       totalFilesAdded: totals.added,

package/src/ingestion/types.ts

@@ -12,8 +12,14 @@ import type {
   RecordAttachmentInventoryItem,
 } from "../converters/types";
 import type { EgressLineage } from "../core/egress-provenance";
+import type { ChunkingPolicyToken } from "../store/chunking";
 import type { DirectoryAvailabilityPort } from "./source-availability/types";
 
+import {
+  DEFAULT_CHUNKING_PARAMS,
+  type ChunkingParams,
+} from "../config/chunking";
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Walker Types
 // ─────────────────────────────────────────────────────────────────────────────
@@ -115,8 +121,7 @@ export interface ChunkParams {
 
 /** Default chunk params */
 export const DEFAULT_CHUNK_PARAMS: ChunkParams = {
-  maxTokens: 800,
-  overlapPercent: 0.15,
+  ...DEFAULT_CHUNKING_PARAMS,
 };
 
 /** Chunked output */
@@ -157,6 +162,10 @@ export interface ChunkerPort {
 
 /** Sync options */
 export interface SyncOptions {
+  /** Index-wide configured policy; omitted means the existing defaults. */
+  chunking?: Partial<ChunkingParams>;
+  /** Internal token passed from an outer sync; never a public CLI override. */
+  chunkingToken?: ChunkingPolicyToken;
   /** Run git pull before scanning */
   gitPull?: boolean;
   /** Run collection updateCmd before scanning */
@@ -266,6 +275,8 @@ export interface FileSyncResult {
 
 /** Collection sync summary */
 export interface CollectionSyncResult {
+  /** Cached layouts updated independently of source-refresh file counters. */
+  rechunkedMirrors?: number;
   collection: string;
   filesProcessed: number;
   filesAdded: number;
@@ -285,6 +296,8 @@ export interface CollectionSyncResult {
 
 /** Full sync summary */
 export interface SyncResult {
+  /** Cached layouts updated independently of source-refresh file counters. */
+  rechunkedMirrors?: number;
   collections: CollectionSyncResult[];
   totalDurationMs: number;
   totalFilesProcessed: number;

package/src/mcp/tools/status.ts

@@ -8,6 +8,7 @@ import type { IndexStatus } from "../../store/types";
 import type { ToolContext } from "../server";
 
 import { buildContentTypeBoostStatus } from "../../config/content-types";
+import { formatChunkingStatus } from "../../core/chunking-status";
 import { resolveModelUri } from "../../llm/registry";
 import { createStandaloneResidentStatus } from "../../serve/resident-status";
 import { runTool, type ToolResult } from "./index";
@@ -55,6 +56,9 @@ function formatStatus(status: IndexStatus): string {
     lines.push(`Embedding backlog: ${status.embeddingBacklog} chunks`);
   }
 
+  const chunking = formatChunkingStatus(status.chunking);
+  if (chunking) lines.push(chunking);
+
   if (status.recentErrors > 0) {
     lines.push(`Recent errors: ${status.recentErrors} (last 24h)`);
   }
@@ -79,6 +83,7 @@ export function handleStatus(
     async () => {
       const result = await ctx.store.getStatus({
         embedModel: resolveModelUri(ctx.config, "embed"),
+        chunking: ctx.config.chunking ?? {},
       });
       if (!result.ok) {
         throw new Error(result.error.message);

package/src/sdk/client.ts

@@ -1352,6 +1352,7 @@ class GnoClientImpl implements GnoClient {
     const status = unwrapStore(
       await this.store.getStatus({
         embedModel: resolveModelUri(this.config, "embed"),
+        chunking: this.config.chunking ?? {},
       })
     );
     return {

package/src/serve/status-model.ts

@@ -1,5 +1,6 @@
 import type { ContentTypeBoostStatus } from "../config/content-types";
 import type { ActivationStatus } from "../core/activation-status";
+import type { ChunkingStatus } from "../store/chunking";
 
 export type HealthCheckStatus = "ok" | "warn" | "error";
 
@@ -176,6 +177,7 @@ export interface BootstrapState {
 }
 
 export interface AppStatusResponse {
+  chunking?: ChunkingStatus;
   resident: ResidentStatus;
   indexName: string;
   configPath: string;

package/src/serve/status.ts

@@ -654,6 +654,7 @@ export async function buildAppStatus(
 ): Promise<AppStatusResponse> {
   const result = await ctx.store.getStatus({
     embedModel: resolveModelUri(ctx.config, "embed"),
+    chunking: ctx.config.chunking ?? {},
   });
   if (!result.ok) {
     throw result.error;
@@ -733,6 +734,7 @@ export async function buildAppStatus(
     totalDocuments: status.activeDocuments,
     totalChunks: status.totalChunks,
     embeddingBacklog: status.embeddingBacklog,
+    chunking: status.chunking,
     lastUpdated: status.lastUpdatedAt,
     recentErrors: status.recentErrors,
     healthy: checks.every((check) => check.status === "ok"),

package/src/store/chunking.ts

@@ -0,0 +1,31 @@
+import type { ChunkingParams } from "../config/chunking";
+
+export interface ChunkingPolicyToken {
+  params: ChunkingParams;
+  generation: number;
+}
+
+export interface PendingChunkingMirror {
+  mirrorHash: string;
+  sourcePath: string;
+  languageHint?: string;
+}
+
+export interface ChunkingStatus {
+  configured: ChunkingParams;
+  applied: ChunkingParams | null;
+  state: "empty" | "legacy-default" | "current" | "pending" | "mixed";
+  pendingDocuments: number;
+  pendingMirrors: number;
+}
+
+export class ChunkingPolicyConflictError extends Error {
+  readonly code = "CHUNKING_POLICY_CONFLICT";
+
+  constructor() {
+    super(
+      "CHUNKING_POLICY_CONFLICT: The index chunking policy changed. Reopen the client or restart the resident runtime with the intended configuration."
+    );
+    this.name = "ChunkingPolicyConflictError";
+  }
+}

package/src/store/sqlite/adapter.ts

@@ -113,6 +113,11 @@ import type {
 import type { SqliteDbProvider } from "./types";
 
 import { buildUri, deriveDocid, stripUriIndex } from "../../app/constants";
+import {
+  DEFAULT_CHUNKING_PARAMS,
+  resolveChunkingParams,
+  type ChunkingParams,
+} from "../../config/chunking";
 import {
   type Collection,
   type Context,
@@ -140,6 +145,11 @@ import {
   parseActivationReceipt,
   serializeActivationReceipt,
 } from "../activation-receipts";
+import {
+  ChunkingPolicyConflictError,
+  type ChunkingPolicyToken,
+  type PendingChunkingMirror,
+} from "../chunking";
 import { getSchemaVersion, migrations, runMigrations } from "../migrations";
 import { err, ok } from "../types";
 import { getStoredEmbeddingFingerprint } from "../vector/freshness";
@@ -164,6 +174,15 @@ import {
   purgeDocumentChanges as purgeStoredDocumentChanges,
   snapshotDocumentChange,
 } from "./change-journal-store";
+import {
+  assertChunkingTarget,
+  claimChunkingTarget,
+  getChunkingStatus,
+  markChunkingApplied,
+  pendingChunkingMirrors,
+  pruneChunkingMetadata,
+  readChunkingTarget,
+} from "./chunking-policy";
 import {
   appendEgressAuditReceipt as appendStoredEgressAuditReceipt,
   appendEgressAuditReceiptWithRetention as appendStoredEgressAuditReceiptWithRetention,
@@ -535,6 +554,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   private shutdownFenced = false;
   private shutdownDeadline?: number;
   private contextGeneration = 0;
+  private chunkingGeneration = 0;
 
   // ─────────────────────────────────────────────────────────────────────────
   // Lifecycle
@@ -603,6 +623,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         return result;
       }
 
+      this.chunkingGeneration = readChunkingTarget(this.db).generation;
       this.contextGeneration += 1;
       return result;
     } catch (cause) {
@@ -2631,12 +2652,20 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
 
   async upsertChunks(
     mirrorHash: string,
-    chunks: ChunkInput[]
+    chunks: ChunkInput[],
+    policy?: ChunkingPolicyToken
   ): Promise<StoreResult<void>> {
     try {
       const db = this.ensureOpen();
 
       const transaction = db.transaction(() => {
+        assertChunkingTarget(
+          db,
+          policy ?? {
+            params: DEFAULT_CHUNKING_PARAMS,
+            generation: this.chunkingGeneration,
+          }
+        );
         // Retain stable rows: DELETE cascades erase valid legacy vectors even
         // when duplicate ingestion produces exactly the same embedding input.
         const nextBySequence = new Map(
@@ -2693,13 +2722,83 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       return ok(undefined);
     } catch (cause) {
       return err(
-        "QUERY_FAILED",
+        cause instanceof ChunkingPolicyConflictError
+          ? cause.code
+          : "QUERY_FAILED",
         cause instanceof Error ? cause.message : "Failed to upsert chunks",
         cause
       );
     }
   }
 
+  async claimChunkingPolicy(
+    input: ChunkingParams
+  ): Promise<StoreResult<ChunkingPolicyToken>> {
+    const params = resolveChunkingParams(input);
+    const result = await this.withTransaction(async () =>
+      claimChunkingTarget(this.ensureOpen(), this.chunkingGeneration, params)
+    );
+    if (result.ok) this.chunkingGeneration = result.value.generation;
+    else if (result.error.cause instanceof ChunkingPolicyConflictError) {
+      return err(
+        "CHUNKING_POLICY_CONFLICT",
+        result.error.message,
+        result.error.cause
+      );
+    }
+    return result;
+  }
+
+  async listPendingChunkingMirrors(
+    policy: ChunkingPolicyToken,
+    afterHash = ""
+  ): Promise<StoreResult<PendingChunkingMirror[]>> {
+    try {
+      return ok(pendingChunkingMirrors(this.ensureOpen(), policy, afterHash));
+    } catch (cause) {
+      return err(
+        cause instanceof ChunkingPolicyConflictError
+          ? cause.code
+          : "QUERY_FAILED",
+        cause instanceof Error
+          ? cause.message
+          : "Failed to list pending chunk layouts",
+        cause
+      );
+    }
+  }
+
+  async applyChunkLayout(
+    mirrorHash: string,
+    chunks: ChunkInput[],
+    policy: ChunkingPolicyToken,
+    sourcePath: string,
+    languageHint?: string
+  ): Promise<StoreResult<void>> {
+    const result = await this.withTransaction(async () => {
+      const db = this.ensureOpen();
+      assertChunkingTarget(db, policy);
+      const applied = await this.upsertChunks(mirrorHash, chunks, policy);
+      if (!applied.ok)
+        throw applied.error.cause ?? new Error(applied.error.message);
+      const indexed = await this.rebuildFtsForHash(mirrorHash);
+      if (!indexed.ok)
+        throw indexed.error.cause ?? new Error(indexed.error.message);
+      markChunkingApplied(db, mirrorHash, policy, sourcePath, languageHint);
+    });
+    if (
+      !result.ok &&
+      result.error.cause instanceof ChunkingPolicyConflictError
+    ) {
+      return err(
+        "CHUNKING_POLICY_CONFLICT",
+        result.error.message,
+        result.error.cause
+      );
+    }
+    return result;
+  }
+
   async getChunks(mirrorHash: string): Promise<StoreResult<ChunkRow[]>> {
     try {
       const db = this.ensureOpen();
@@ -5690,6 +5789,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   async getStatus(options?: {
     embedModel?: string;
     embedFingerprint?: string;
+    chunking?: Partial<ChunkingParams>;
   }): Promise<StoreResult<IndexStatus>> {
     try {
       const db = this.ensureOpen();
@@ -5884,6 +5984,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         activeDocuments: totalsRow?.active ?? 0,
         totalChunks: chunkCount,
         embeddingBacklog: variantStatus?.backlog ?? backlogRow?.count ?? 0,
+        chunking: getChunkingStatus(db, options?.chunking),
         recentErrors,
         lastUpdatedAt: lastUpdatedRow?.last_updated ?? null,
         healthy,
@@ -5968,6 +6069,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           )
         `);
         orphanedContent = contentResult.changes;
+        pruneChunkingMetadata(db);
 
         // Delete chunks for deleted content
         const chunksResult = db.run(`