@diegoaltoworks/chatter 2.0.0 → 2.1.0

package/dist/core/chunking.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Section-aware Markdown chunking.
+ *
+ * A pure line scanner (no Markdown parser dependency) that splits a document at
+ * every ATX heading, records the heading trail each chunk sat under, and keeps
+ * fenced code blocks, lists and tables whole. A section longer than the cap is
+ * split on blank-line paragraph boundaries; consecutive small sections are
+ * never merged.
+ */
+/** One chunk of a document, in reading order. */
+export type SectionChunk = {
+    /** The chunk's text. A section's first chunk starts with its heading line. */
+    text: string;
+    /** Heading trail from the H1 down; empty for text before the first heading. */
+    section: string[];
+    /** Zero-based index of the chunk within the document. */
+    position: number;
+};
+/**
+ * Split Markdown `text` into structure-aware chunks of at most `max` characters
+ * where the structure allows it.
+ *
+ * Every ATX heading starts a new chunk, with the heading line kept at its top.
+ * A section longer than `max` is split on paragraph boundaries, each piece
+ * keeping the section's trail. A fenced code block, list or table is atomic and
+ * is emitted whole even if it alone exceeds `max`. Empty chunks are never
+ * produced, and every non-blank input line appears in exactly one chunk, in
+ * order.
+ */
+export declare function chunkSections(text: string, max?: number): SectionChunk[];
+//# sourceMappingURL=chunking.d.ts.map

package/dist/core/chunking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/core/chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,iDAAiD;AACjD,MAAM,MAAM,YAAY,GAAG;IACzB,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AA+GF;;;;;;;;;;GAUG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,SAAM,GAAG,YAAY,EAAE,CA6BrE"}

package/dist/core/retrieval.d.ts

@@ -16,6 +16,13 @@ export declare const DEFAULT_KNOWLEDGE_DIR = "./config/knowledge";
 export interface Retriever {
     /** Retrieve up to `k` chunks across `allowedBuckets` for `query`, most relevant first. */
     query(query: string, k: number, allowedBuckets: string[]): Promise<string[]>;
+    /**
+     * Optional record-returning variant of `query`, for a host that needs to
+     * know where a passage came from (a search route, citations). `prepareChat`
+     * never calls it; implement it only if your own code wants provenance.
+     * Same scoping and ordering rules as `query`.
+     */
+    queryChunks?(query: string, k: number, allowedBuckets: string[]): Promise<RetrievedChunk[]>;
     /**
      * Optional one-time ingest/warm-up step, run once at server startup before
      * the store answers any query. Omit it for a retriever that is always
@@ -23,6 +30,18 @@ export interface Retriever {
      */
     build?(): Promise<void>;
 }
+/** One retrieval result with its provenance. See {@link Retriever.queryChunks}. */
+export interface RetrievedChunk {
+    /** The chunk text alone, without any embedding context line. */
+    text: string;
+    bucket: string;
+    /** Where the chunk came from; knowledgeDir-relative for rows ingested in `'sections'` mode. */
+    source: string;
+    /** Heading trail from the H1 down; empty when the row has none (every `'lines'` row). */
+    section: string[];
+    /** Cosine similarity between the query and the chunk embedding. */
+    score: number;
+}
 /**
  * Embeds a batch of texts into vectors, in input order. Lets
  * {@link VectorStore} stay decoupled from any specific embeddings provider -
@@ -52,6 +71,8 @@ export declare function openLibsqlClient(database: {
     url: string;
     authToken: string;
 }): Promise<LibsqlClient>;
+/** Chunking strategy for {@link VectorStoreOptions.chunking}. */
+export type ChunkingMode = "lines" | "sections";
 /**
  * `VectorStore` always takes an already-open libsql client rather than
  * credentials to open its own - the same rule every other store in this
@@ -68,6 +89,15 @@ export interface VectorStoreOptions {
     databaseClient: LibsqlClient;
     /** Directory of markdown knowledge files. Default: `./config/knowledge` */
     knowledgeDir?: string;
+    /**
+     * How knowledge files are cut into chunks. `'lines'` (default) keeps the
+     * original fixed-size line chunks, so ids and embeddings are unchanged.
+     * `'sections'` cuts at Markdown headings (see
+     * [chunking.ts](./chunking.ts)), records each chunk's heading trail and
+     * position, and embeds it with a `<source> > <heading trail>` context line.
+     * Switching modes re-embeds every chunk once on the next build.
+     */
+    chunking?: ChunkingMode;
     /** Logger for build progress. Default: a console logger writing to stderr. */
     logger?: Logger;
     /**
@@ -92,6 +122,7 @@ export declare class VectorStore implements Retriever {
      */
     readonly db: LibsqlClient;
     private knowledgeDir;
+    private chunking;
     private logger;
     private buildLock;
     private instanceId;
@@ -113,6 +144,12 @@ export declare class VectorStore implements Retriever {
     private renewBuildLock;
     /** The ingest itself. Only ever runs with the build lock held. */
     private buildUnderLock;
+    /**
+     * Adds the nullable `section`/`position` columns to a `chunks` table that
+     * every released version created without them. Checked via table_info so it
+     * is a no-op once present; only `'sections'` mode ever calls it.
+     */
+    private ensureSectionColumns;
     private cleanupStaleChunks;
     private static cosine;
     /**
@@ -126,6 +163,8 @@ export declare class VectorStore implements Retriever {
      * interpolated. An empty list retrieves nothing, and short-circuits before
      * the embedding call.
      */
+    queryChunks(q: string, k?: number, allowed?: string[]): Promise<RetrievedChunk[]>;
+    /** Same results as {@link queryChunks}, as bare text. */
     query(q: string, k?: number, allowed?: string[]): Promise<string[]>;
 }
 //# sourceMappingURL=retrieval.d.ts.map

package/dist/core/retrieval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"retrieval.d.ts","sourceRoot":"","sources":["../../src/core/retrieval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,aAAa,CAAC;AAErB,OAAO,EAAuB,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AAI5D,kFAAkF;AAClF,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,0FAA0F;IAC1F,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E;;;;OAIG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAEhE;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAK7D;AAED,2KAA2K;AAC3K,eAAO,MAAM,gCAAgC,QAG4B,CAAC;AAE1E,0MAA0M;AAC1M,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAO5D;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,YAAY,CAAC,CAKxB;AAuBD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,uEAAuE;IACvE,cAAc,EAAE,YAAY,CAAC;IAC7B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2HAA2H;IAC3H,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,WAAY,YAAW,SAAS;IAczC,OAAO,CAAC,KAAK;IAbf;;;;OAIG;IACH,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC;IAC1B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IAEjC,YACU,KAAK,EAAE,QAAQ,EACvB,OAAO,EAAE,kBAAkB,EAQ5B;IAED;;;;;;;;;;OAUG;IACG,KAAK,kBAyBV;IAED,0FAA0F;IAC1F,OAAO,CAAC,cAAc;IAStB,kEAAkE;YACpD,cAAc;YA8Gd,kBAAkB;IAqChC,OAAO,CAAC,MAAM,CAAC,MAAM;IAYrB;;;;;;;;;;OAUG;IACG,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,SAAI,EAAE,OAAO,GAAE,MAAM,EAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB7E;CACF"}
+{"version":3,"file":"retrieval.d.ts","sourceRoot":"","sources":["../../src/core/retrieval.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAuB,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AAI5D,kFAAkF;AAClF,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,0FAA0F;IAC1F,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E;;;;;OAKG;IACH,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAC5F;;;;OAIG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,mFAAmF;AACnF,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,+FAA+F;IAC/F,MAAM,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,mEAAmE;IACnE,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAEhE;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAK7D;AAED,2KAA2K;AAC3K,eAAO,MAAM,gCAAgC,QAG4B,CAAC;AAE1E,0MAA0M;AAC1M,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAO5D;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,YAAY,CAAC,CAKxB;AAuBD,iEAAiE;AACjE,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,UAAU,CAAC;AAahD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,uEAAuE;IACvE,cAAc,EAAE,YAAY,CAAC;IAC7B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2HAA2H;IAC3H,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,WAAY,YAAW,SAAS;IAezC,OAAO,CAAC,KAAK;IAdf;;;;OAIG;IACH,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC;IAC1B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAe;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IAEjC,YACU,KAAK,EAAE,QAAQ,EACvB,OAAO,EAAE,kBAAkB,EAS5B;IAED;;;;;;;;;;OAUG;IACG,KAAK,kBAyBV;IAED,0FAA0F;IAC1F,OAAO,CAAC,cAAc;IAStB,kEAAkE;YACpD,cAAc;IAkJ5B;;;;OAIG;YACW,oBAAoB;YASpB,kBAAkB;IAqChC,OAAO,CAAC,MAAM,CAAC,MAAM;IAYrB;;;;;;;;;;OAUG;IACG,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,SAAI,EAAE,OAAO,GAAE,MAAM,EAAa,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6B3F;IAED,yDAAyD;IACnD,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,SAAI,EAAE,OAAO,GAAE,MAAM,EAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAE7E;CACF"}

package/dist/index.d.ts

@@ -20,6 +20,8 @@ export type { BucketsFor, BucketsForContext } from "./core/buckets";
 export { defaultBuckets, resolveBuckets } from "./core/buckets";
 export type { BuildLock, BuildLockRow } from "./core/buildLock";
 export { BUILD_LOCK_KEY, BUILD_LOCK_STALE_MS, canAcquireBuildLock, createTursoBuildLock, } from "./core/buildLock";
+export type { SectionChunk } from "./core/chunking";
+export { chunkSections } from "./core/chunking";
 export { DEFAULT_REFUSAL, detectLeakage, guardOutput, scrubOutput } from "./core/guardrails";
 export { completeOnce, completeStream, DEFAULT_MODEL } from "./core/llm";
 export { loadKnowledge } from "./core/loaders";
@@ -30,7 +32,7 @@ export { normalizeChatBody, normalizeMessages } from "./core/messages";
 export type { FallbackContext, FallbackFn, PipelineMessage, PipelineMode, PreparedChat, RerankContext, RerankContextArgs, RewriteQuery, RewriteQueryArgs, } from "./core/pipeline";
 export { prepareChat } from "./core/pipeline";
 export { DEFAULT_PROMPTS_DIR, PromptLoader } from "./core/prompts";
-export type { Embedder, Retriever, VectorStoreOptions } from "./core/retrieval";
+export type { ChunkingMode, Embedder, RetrievedChunk, Retriever, VectorStoreOptions, } from "./core/retrieval";
 export { createOpenAIEmbedder, DEFAULT_KNOWLEDGE_DIR, VectorStore } from "./core/retrieval";
 export { createSession, getActiveSessions, getSessionInfo, validateSession, } from "./core/session";
 export type { MCPServerOptions, MCPTransportMode } from "./mcp-server";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAIvC,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,QAAQ,EACR,eAAe,GAChB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,QAAQ,EACR,aAAa,EACb,cAAc,EACd,aAAa,EACb,WAAW,EACX,cAAc,EACd,mBAAmB,GACpB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAC9E,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEpE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhE,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAG1D,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,YAAY,EACV,eAAe,EACf,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEhF,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEvE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,YAAY,EACV,WAAW,EACX,aAAa,EACb,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,GACnB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAIvC,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,QAAQ,EACR,eAAe,GAChB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,QAAQ,EACR,aAAa,EACb,cAAc,EACd,aAAa,EACb,WAAW,EACX,cAAc,EACd,mBAAmB,GACpB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAC9E,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEpE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhE,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAG1D,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,YAAY,EACV,eAAe,EACf,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnE,YAAY,EACV,YAAY,EACZ,QAAQ,EACR,cAAc,EACd,SAAS,EACT,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEvE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,YAAY,EACV,WAAW,EACX,aAAa,EACb,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,GACnB,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -94,6 +94,126 @@ var init_buildLock = __esm({
   }
 });
 
+// src/core/chunking.ts
+function headingOf(line) {
+  const match = HEADING.exec(line);
+  if (!match) return void 0;
+  const title = (match[2] ?? "").replace(/[ \t]+#+$/, "").replace(/^#+$/, "").trim();
+  return { level: match[1].length, title };
+}
+function startsTable(lines, i) {
+  return lines[i].includes("|") && i + 1 < lines.length && TABLE_DELIMITER.test(lines[i + 1]);
+}
+function fenceEnd(lines, start) {
+  const opener = FENCE.exec(lines[start]);
+  if (!opener) return start + 1;
+  const marker = opener[1];
+  const closer = new RegExp(`^ {0,3}${marker[0] === "`" ? "`" : "~"}{${marker.length},}[ \\t]*$`);
+  for (let i = start + 1; i < lines.length; i++) if (closer.test(lines[i])) return i + 1;
+  return lines.length;
+}
+function listEnd(lines, start) {
+  let end = start + 1;
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (isBlank(line)) {
+      let next = i + 1;
+      while (next < lines.length && isBlank(lines[next])) next++;
+      if (next < lines.length && (LIST_ITEM.test(lines[next]) || isIndented(lines[next]))) {
+        i = next - 1;
+        continue;
+      }
+      break;
+    }
+    if (headingOf(line) && !isIndented(line)) break;
+    if (FENCE.test(line) && !isIndented(line)) break;
+    end = i + 1;
+  }
+  return end;
+}
+function tableEnd(lines, start) {
+  let end = start + 2;
+  while (end < lines.length && !isBlank(lines[end]) && lines[end].includes("|")) end++;
+  return end;
+}
+function paragraphEnd(lines, start) {
+  let end = start + 1;
+  while (end < lines.length && !isBlank(lines[end]) && !headingOf(lines[end]) && !FENCE.test(lines[end]) && !LIST_ITEM.test(lines[end])) {
+    end++;
+  }
+  return end;
+}
+function scan(text) {
+  const lines = text.replace(/\r\n?/g, "\n").split("\n");
+  const sections = [{ trail: [], blocks: [] }];
+  const stack = [];
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i];
+    if (isBlank(line)) {
+      i++;
+      continue;
+    }
+    const heading = FENCE.test(line) ? void 0 : headingOf(line);
+    if (heading) {
+      while (stack.length && stack[stack.length - 1].level >= heading.level) stack.pop();
+      stack.push(heading);
+      sections.push({ trail: stack.map((h) => h.title), heading: line, blocks: [] });
+      i++;
+      continue;
+    }
+    let end;
+    if (FENCE.test(line)) end = fenceEnd(lines, i);
+    else if (LIST_ITEM.test(line)) end = listEnd(lines, i);
+    else if (startsTable(lines, i)) end = tableEnd(lines, i);
+    else end = paragraphEnd(lines, i);
+    sections[sections.length - 1].blocks.push({ lines: lines.slice(i, end) });
+    i = end;
+  }
+  return sections;
+}
+function chunkSections(text, max = 900) {
+  const cap = Math.max(1, max);
+  const out = [];
+  const emit = (parts, section) => {
+    const body = parts.join("\n\n");
+    if (body.trim() === "") return;
+    out.push({ text: body, section: [...section], position: out.length });
+  };
+  for (const section of scan(text)) {
+    let parts = section.heading ? [section.heading.trim()] : [];
+    let size = parts.join("\n\n").length;
+    let hasBlock = false;
+    for (const block of section.blocks) {
+      const piece = block.lines.join("\n");
+      const added = piece.length + (parts.length ? 2 : 0);
+      if (hasBlock && size + added > cap) {
+        emit(parts, section.trail);
+        parts = [];
+        size = 0;
+        hasBlock = false;
+      }
+      size += piece.length + (parts.length ? 2 : 0);
+      parts.push(piece);
+      hasBlock = true;
+    }
+    emit(parts, section.trail);
+  }
+  return out;
+}
+var FENCE, HEADING, LIST_ITEM, TABLE_DELIMITER, isBlank, isIndented;
+var init_chunking = __esm({
+  "src/core/chunking.ts"() {
+    "use strict";
+    FENCE = /^ {0,3}(`{3,}|~{3,})/;
+    HEADING = /^ {0,3}(#{1,6})(?:[ \t]+(.*?))?[ \t]*$/;
+    LIST_ITEM = /^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+\S/;
+    TABLE_DELIMITER = /^[ \t]*\|?[ \t]*:?-+:?[ \t]*(?:\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$/;
+    isBlank = (line) => line.trim() === "";
+    isIndented = (line) => /^(?: {2,}|\t)/.test(line);
+  }
+});
+
 // src/core/loaders.ts
 function walk(dir) {
   const items = [];
@@ -199,11 +319,13 @@ async function sha256(input) {
   const hash = await crypto.subtle.digest("SHA-256", data);
   return Array.from(new Uint8Array(hash)).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
-var EMB_MODEL, DEFAULT_KNOWLEDGE_DIR, DATABASE_CONFIG_REQUIRED_MESSAGE, VectorStore;
+var import_node_path3, EMB_MODEL, DEFAULT_KNOWLEDGE_DIR, DATABASE_CONFIG_REQUIRED_MESSAGE, VectorStore;
 var init_retrieval = __esm({
   "src/core/retrieval.ts"() {
     "use strict";
+    import_node_path3 = require("node:path");
     init_buildLock();
+    init_chunking();
     init_loaders();
     init_logger();
     EMB_MODEL = "text-embedding-3-large";
@@ -214,6 +336,7 @@ var init_retrieval = __esm({
         this.embed = embed;
         this.db = options.databaseClient;
         this.knowledgeDir = options.knowledgeDir || DEFAULT_KNOWLEDGE_DIR;
+        this.chunking = options.chunking ?? "lines";
         this.logger = options.logger ?? createConsoleLogger();
         this.buildLock = options.buildLock ?? createTursoBuildLock(this.db);
         this.instanceId = options.instanceId ?? crypto.randomUUID();
@@ -227,6 +350,7 @@ var init_retrieval = __esm({
        */
       db;
       knowledgeDir;
+      chunking;
       logger;
       buildLock;
       instanceId;
@@ -287,11 +411,41 @@ var init_retrieval = __esm({
           this.logger.info("No knowledge documents and no existing chunks - nothing to build");
           return;
         }
+        this.logger.info(`Chunking mode: ${this.chunking}`);
+        if (this.chunking === "sections") await this.ensureSectionColumns();
         const rows = [];
         for (const d of docs) {
-          for (const part of chunk(d.text)) {
-            const id = await sha256(`${d.bucket}|${d.source}|${part}`);
-            rows.push({ id, bucket: d.bucket, source: d.source, text: part });
+          if (this.chunking === "sections") {
+            const source = (0, import_node_path3.relative)(this.knowledgeDir, d.source).split(import_node_path3.sep).join("/");
+            for (const c of chunkSections(d.text)) {
+              const section = c.section.join(" > ");
+              const id = await sha256(`${d.bucket}|${source}|${section}|${c.text}`);
+              const context = section ? `${source} > ${section}` : source;
+              rows.push({
+                id,
+                bucket: d.bucket,
+                source,
+                text: c.text,
+                embedText: `${context}
+
+${c.text}`,
+                section,
+                position: c.position
+              });
+            }
+          } else {
+            for (const part of chunk(d.text)) {
+              const id = await sha256(`${d.bucket}|${d.source}|${part}`);
+              rows.push({
+                id,
+                bucket: d.bucket,
+                source: d.source,
+                text: part,
+                embedText: part,
+                section: null,
+                position: null
+              });
+            }
           }
         }
         this.logger.info(`Created ${rows.length} chunks from knowledge documents`);
@@ -300,11 +454,17 @@ var init_retrieval = __esm({
         for (let i = 0; i < rows.length; i += UPSERT_BATCH) {
           const batch = rows.slice(i, i + UPSERT_BATCH);
           await this.db.batch(
-            batch.map((r) => ({
-              sql: `INSERT INTO chunks(id,bucket,source,text) VALUES(?,?,?,?)
-                ON CONFLICT(id) DO NOTHING`,
-              args: [r.id, r.bucket, r.source, r.text]
-            })),
+            batch.map(
+              (r) => this.chunking === "sections" ? {
+                sql: `INSERT INTO chunks(id,bucket,source,text,section,position) VALUES(?,?,?,?,?,?)
+                      ON CONFLICT(id) DO NOTHING`,
+                args: [r.id, r.bucket, r.source, r.text, r.section, r.position]
+              } : {
+                sql: `INSERT INTO chunks(id,bucket,source,text) VALUES(?,?,?,?)
+                      ON CONFLICT(id) DO NOTHING`,
+                args: [r.id, r.bucket, r.source, r.text]
+              }
+            ),
             "write"
           );
         }
@@ -325,7 +485,7 @@ var init_retrieval = __esm({
           return;
         }
         this.logger.info(`Embedding ${missing.length} new/updated chunks with ${EMB_MODEL}...`);
-        const textById = new Map(rows.map((r) => [r.id, r.text]));
+        const textById = new Map(rows.map((r) => [r.id, r.embedText]));
         const BATCH = 96;
         for (let i = 0; i < missing.length; i += BATCH) {
           const batchIds = missing.slice(i, i + BATCH);
@@ -346,6 +506,18 @@ var init_retrieval = __esm({
         }
         this.logger.info(`Successfully embedded ${missing.length} new chunks`);
       }
+      /**
+       * Adds the nullable `section`/`position` columns to a `chunks` table that
+       * every released version created without them. Checked via table_info so it
+       * is a no-op once present; only `'sections'` mode ever calls it.
+       */
+      async ensureSectionColumns() {
+        const info = await this.db.execute("PRAGMA table_info(chunks)");
+        const have = new Set(info.rows.map((r) => String(r.name)));
+        if (!have.has("section")) await this.db.execute("ALTER TABLE chunks ADD COLUMN section TEXT");
+        if (!have.has("position"))
+          await this.db.execute("ALTER TABLE chunks ADD COLUMN position INTEGER");
+      }
       // Remove chunks from database that no longer exist in markdown files
       async cleanupStaleChunks(currentIds) {
         const result = await this.db.execute("SELECT id FROM chunks");
@@ -392,25 +564,34 @@ var init_retrieval = __esm({
        * interpolated. An empty list retrieves nothing, and short-circuits before
        * the embedding call.
        */
-      async query(q, k = 6, allowed = ["base"]) {
+      async queryChunks(q, k = 6, allowed = ["base"]) {
         if (allowed.length === 0) return [];
         const [qv] = await this.embed([q]);
         const placeholders = allowed.map(() => "?").join(",");
         const res = await this.db.execute({
-          sql: `SELECT c.id, c.text, e.embedding
+          sql: `SELECT c.*, e.embedding
             FROM chunks c
             JOIN embeddings e ON e.id = c.id
             WHERE c.bucket IN (${placeholders})`,
           args: allowed
         });
-        const scored = [];
-        for (const row of res.rows) {
+        const scored = res.rows.map((row) => {
           const emb = JSON.parse(String(row.embedding));
-          const s = _VectorStore.cosine(qv, emb);
-          scored.push({ s, text: String(row.text) });
-        }
-        scored.sort((a, b) => b.s - a.s);
-        return scored.slice(0, k).map((r) => r.text);
+          const trail = row.section == null ? "" : String(row.section);
+          return {
+            text: String(row.text),
+            bucket: String(row.bucket ?? ""),
+            source: String(row.source ?? ""),
+            section: trail ? trail.split(" > ") : [],
+            score: _VectorStore.cosine(qv, emb)
+          };
+        });
+        scored.sort((a, b) => b.score - a.score);
+        return scored.slice(0, k);
+      }
+      /** Same results as {@link queryChunks}, as bare text. */
+      async query(q, k = 6, allowed = ["base"]) {
+        return (await this.queryChunks(q, k, allowed)).map((c) => c.text);
       }
     };
   }
@@ -546,6 +727,7 @@ __export(index_exports, {
   applyTransformReply: () => applyTransformReply,
   canAcquireBuildLock: () => canAcquireBuildLock,
   chatBodyLimit: () => chatBodyLimit,
+  chunkSections: () => chunkSections,
   completeOnce: () => completeOnce,
   completeStream: () => completeStream,
   cors: () => cors,
@@ -866,6 +1048,7 @@ async function resolveBuckets({
 
 // src/index.ts
 init_buildLock();
+init_chunking();
 init_loaders();
 init_logger();
 
@@ -1354,6 +1537,7 @@ async function createMCPServer(config) {
     store = new VectorStore2(createOpenAIEmbedder2(client), {
       databaseClient: db,
       knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
+      chunking: config.chunking,
       logger: log
     });
   }
@@ -2401,7 +2585,7 @@ function publicRoutes(deps) {
 }
 
 // src/server.ts
-var import_node_path4 = require("node:path");
+var import_node_path5 = require("node:path");
 var import_hono5 = require("hono");
 var import_openai2 = __toESM(require("openai"));
 
@@ -2505,7 +2689,7 @@ function loadServeStatic(runtime = detectRuntime()) {
 
 // src/core/widgets.ts
 var import_node_fs3 = require("node:fs");
-var import_node_path3 = require("node:path");
+var import_node_path4 = require("node:path");
 var import_node_url = require("node:url");
 var import_meta = {};
 function resolveStatic(configStaticDir) {
@@ -2514,15 +2698,15 @@ function resolveStatic(configStaticDir) {
   }
   try {
     if (import_meta.url) {
-      const moduleDir = (0, import_node_path3.dirname)((0, import_node_url.fileURLToPath)(import_meta.url));
-      const staticDir = (0, import_node_path3.join)(moduleDir, "static");
+      const moduleDir = (0, import_node_path4.dirname)((0, import_node_url.fileURLToPath)(import_meta.url));
+      const staticDir = (0, import_node_path4.join)(moduleDir, "static");
       if ((0, import_node_fs3.existsSync)(staticDir)) return { staticDir };
     }
   } catch {
   }
   try {
     if (typeof __dirname !== "undefined") {
-      const staticDir = (0, import_node_path3.join)(__dirname, "static");
+      const staticDir = (0, import_node_path4.join)(__dirname, "static");
       if ((0, import_node_fs3.existsSync)(staticDir)) return { staticDir };
     }
   } catch {
@@ -2561,6 +2745,7 @@ async function createServer(config) {
     store = new VectorStore2(createOpenAIEmbedder2(client), {
       databaseClient: db,
       knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
+      chunking: config.chunking,
       logger
     });
   }
@@ -2611,7 +2796,7 @@ async function createServer(config) {
     }
   }
   if (serveStatic && staticDir) {
-    const relativePath = (0, import_node_path4.relative)(process.cwd(), staticDir);
+    const relativePath = (0, import_node_path5.relative)(process.cwd(), staticDir);
     app.get("/chatter.js", serveStatic({ path: `${relativePath}/chatter.js` }));
     app.get("/chatter.css", serveStatic({ path: `${relativePath}/chatter.css` }));
   }
@@ -2773,6 +2958,7 @@ function createShortener(config, fetchImpl = fetch) {
   applyTransformReply,
   canAcquireBuildLock,
   chatBodyLimit,
+  chunkSections,
   completeOnce,
   completeStream,
   cors,