@diegoaltoworks/chatter 1.0.0 → 2.1.0

package/dist/index.mjs

@@ -72,6 +72,126 @@ var init_buildLock = __esm({
   }
 });
 
+// src/core/chunking.ts
+function headingOf(line) {
+  const match = HEADING.exec(line);
+  if (!match) return void 0;
+  const title = (match[2] ?? "").replace(/[ \t]+#+$/, "").replace(/^#+$/, "").trim();
+  return { level: match[1].length, title };
+}
+function startsTable(lines, i) {
+  return lines[i].includes("|") && i + 1 < lines.length && TABLE_DELIMITER.test(lines[i + 1]);
+}
+function fenceEnd(lines, start) {
+  const opener = FENCE.exec(lines[start]);
+  if (!opener) return start + 1;
+  const marker = opener[1];
+  const closer = new RegExp(`^ {0,3}${marker[0] === "`" ? "`" : "~"}{${marker.length},}[ \\t]*$`);
+  for (let i = start + 1; i < lines.length; i++) if (closer.test(lines[i])) return i + 1;
+  return lines.length;
+}
+function listEnd(lines, start) {
+  let end = start + 1;
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (isBlank(line)) {
+      let next = i + 1;
+      while (next < lines.length && isBlank(lines[next])) next++;
+      if (next < lines.length && (LIST_ITEM.test(lines[next]) || isIndented(lines[next]))) {
+        i = next - 1;
+        continue;
+      }
+      break;
+    }
+    if (headingOf(line) && !isIndented(line)) break;
+    if (FENCE.test(line) && !isIndented(line)) break;
+    end = i + 1;
+  }
+  return end;
+}
+function tableEnd(lines, start) {
+  let end = start + 2;
+  while (end < lines.length && !isBlank(lines[end]) && lines[end].includes("|")) end++;
+  return end;
+}
+function paragraphEnd(lines, start) {
+  let end = start + 1;
+  while (end < lines.length && !isBlank(lines[end]) && !headingOf(lines[end]) && !FENCE.test(lines[end]) && !LIST_ITEM.test(lines[end])) {
+    end++;
+  }
+  return end;
+}
+function scan(text) {
+  const lines = text.replace(/\r\n?/g, "\n").split("\n");
+  const sections = [{ trail: [], blocks: [] }];
+  const stack = [];
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i];
+    if (isBlank(line)) {
+      i++;
+      continue;
+    }
+    const heading = FENCE.test(line) ? void 0 : headingOf(line);
+    if (heading) {
+      while (stack.length && stack[stack.length - 1].level >= heading.level) stack.pop();
+      stack.push(heading);
+      sections.push({ trail: stack.map((h) => h.title), heading: line, blocks: [] });
+      i++;
+      continue;
+    }
+    let end;
+    if (FENCE.test(line)) end = fenceEnd(lines, i);
+    else if (LIST_ITEM.test(line)) end = listEnd(lines, i);
+    else if (startsTable(lines, i)) end = tableEnd(lines, i);
+    else end = paragraphEnd(lines, i);
+    sections[sections.length - 1].blocks.push({ lines: lines.slice(i, end) });
+    i = end;
+  }
+  return sections;
+}
+function chunkSections(text, max = 900) {
+  const cap = Math.max(1, max);
+  const out = [];
+  const emit = (parts, section) => {
+    const body = parts.join("\n\n");
+    if (body.trim() === "") return;
+    out.push({ text: body, section: [...section], position: out.length });
+  };
+  for (const section of scan(text)) {
+    let parts = section.heading ? [section.heading.trim()] : [];
+    let size = parts.join("\n\n").length;
+    let hasBlock = false;
+    for (const block of section.blocks) {
+      const piece = block.lines.join("\n");
+      const added = piece.length + (parts.length ? 2 : 0);
+      if (hasBlock && size + added > cap) {
+        emit(parts, section.trail);
+        parts = [];
+        size = 0;
+        hasBlock = false;
+      }
+      size += piece.length + (parts.length ? 2 : 0);
+      parts.push(piece);
+      hasBlock = true;
+    }
+    emit(parts, section.trail);
+  }
+  return out;
+}
+var FENCE, HEADING, LIST_ITEM, TABLE_DELIMITER, isBlank, isIndented;
+var init_chunking = __esm({
+  "src/core/chunking.ts"() {
+    "use strict";
+    FENCE = /^ {0,3}(`{3,}|~{3,})/;
+    HEADING = /^ {0,3}(#{1,6})(?:[ \t]+(.*?))?[ \t]*$/;
+    LIST_ITEM = /^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+\S/;
+    TABLE_DELIMITER = /^[ \t]*\|?[ \t]*:?-+:?[ \t]*(?:\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$/;
+    isBlank = (line) => line.trim() === "";
+    isIndented = (line) => /^(?: {2,}|\t)/.test(line);
+  }
+});
+
 // src/core/loaders.ts
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
@@ -139,6 +259,7 @@ __export(retrieval_exports, {
   openLibsqlClient: () => openLibsqlClient,
   wrapMissingLibsqlError: () => wrapMissingLibsqlError
 });
+import { relative, sep } from "node:path";
 function createOpenAIEmbedder(client) {
   return async (input) => {
     const res = await client.embeddings.create({ model: EMB_MODEL, input });
@@ -181,6 +302,7 @@ var init_retrieval = __esm({
   "src/core/retrieval.ts"() {
     "use strict";
     init_buildLock();
+    init_chunking();
     init_loaders();
     init_logger();
     EMB_MODEL = "text-embedding-3-large";
@@ -191,6 +313,7 @@ var init_retrieval = __esm({
         this.embed = embed;
         this.db = options.databaseClient;
         this.knowledgeDir = options.knowledgeDir || DEFAULT_KNOWLEDGE_DIR;
+        this.chunking = options.chunking ?? "lines";
         this.logger = options.logger ?? createConsoleLogger();
         this.buildLock = options.buildLock ?? createTursoBuildLock(this.db);
         this.instanceId = options.instanceId ?? crypto.randomUUID();
@@ -204,6 +327,7 @@ var init_retrieval = __esm({
        */
       db;
       knowledgeDir;
+      chunking;
       logger;
       buildLock;
       instanceId;
@@ -264,11 +388,41 @@ var init_retrieval = __esm({
           this.logger.info("No knowledge documents and no existing chunks - nothing to build");
           return;
         }
+        this.logger.info(`Chunking mode: ${this.chunking}`);
+        if (this.chunking === "sections") await this.ensureSectionColumns();
         const rows = [];
         for (const d of docs) {
-          for (const part of chunk(d.text)) {
-            const id = await sha256(`${d.bucket}|${d.source}|${part}`);
-            rows.push({ id, bucket: d.bucket, source: d.source, text: part });
+          if (this.chunking === "sections") {
+            const source = relative(this.knowledgeDir, d.source).split(sep).join("/");
+            for (const c of chunkSections(d.text)) {
+              const section = c.section.join(" > ");
+              const id = await sha256(`${d.bucket}|${source}|${section}|${c.text}`);
+              const context = section ? `${source} > ${section}` : source;
+              rows.push({
+                id,
+                bucket: d.bucket,
+                source,
+                text: c.text,
+                embedText: `${context}
+
+${c.text}`,
+                section,
+                position: c.position
+              });
+            }
+          } else {
+            for (const part of chunk(d.text)) {
+              const id = await sha256(`${d.bucket}|${d.source}|${part}`);
+              rows.push({
+                id,
+                bucket: d.bucket,
+                source: d.source,
+                text: part,
+                embedText: part,
+                section: null,
+                position: null
+              });
+            }
           }
         }
         this.logger.info(`Created ${rows.length} chunks from knowledge documents`);
@@ -277,11 +431,17 @@ var init_retrieval = __esm({
         for (let i = 0; i < rows.length; i += UPSERT_BATCH) {
           const batch = rows.slice(i, i + UPSERT_BATCH);
           await this.db.batch(
-            batch.map((r) => ({
-              sql: `INSERT INTO chunks(id,bucket,source,text) VALUES(?,?,?,?)
-                ON CONFLICT(id) DO NOTHING`,
-              args: [r.id, r.bucket, r.source, r.text]
-            })),
+            batch.map(
+              (r) => this.chunking === "sections" ? {
+                sql: `INSERT INTO chunks(id,bucket,source,text,section,position) VALUES(?,?,?,?,?,?)
+                      ON CONFLICT(id) DO NOTHING`,
+                args: [r.id, r.bucket, r.source, r.text, r.section, r.position]
+              } : {
+                sql: `INSERT INTO chunks(id,bucket,source,text) VALUES(?,?,?,?)
+                      ON CONFLICT(id) DO NOTHING`,
+                args: [r.id, r.bucket, r.source, r.text]
+              }
+            ),
             "write"
           );
         }
@@ -302,7 +462,7 @@ var init_retrieval = __esm({
           return;
         }
         this.logger.info(`Embedding ${missing.length} new/updated chunks with ${EMB_MODEL}...`);
-        const textById = new Map(rows.map((r) => [r.id, r.text]));
+        const textById = new Map(rows.map((r) => [r.id, r.embedText]));
         const BATCH = 96;
         for (let i = 0; i < missing.length; i += BATCH) {
           const batchIds = missing.slice(i, i + BATCH);
@@ -323,6 +483,18 @@ var init_retrieval = __esm({
         }
         this.logger.info(`Successfully embedded ${missing.length} new chunks`);
       }
+      /**
+       * Adds the nullable `section`/`position` columns to a `chunks` table that
+       * every released version created without them. Checked via table_info so it
+       * is a no-op once present; only `'sections'` mode ever calls it.
+       */
+      async ensureSectionColumns() {
+        const info = await this.db.execute("PRAGMA table_info(chunks)");
+        const have = new Set(info.rows.map((r) => String(r.name)));
+        if (!have.has("section")) await this.db.execute("ALTER TABLE chunks ADD COLUMN section TEXT");
+        if (!have.has("position"))
+          await this.db.execute("ALTER TABLE chunks ADD COLUMN position INTEGER");
+      }
       // Remove chunks from database that no longer exist in markdown files
       async cleanupStaleChunks(currentIds) {
         const result = await this.db.execute("SELECT id FROM chunks");
@@ -369,25 +541,34 @@ var init_retrieval = __esm({
        * interpolated. An empty list retrieves nothing, and short-circuits before
        * the embedding call.
        */
-      async query(q, k = 6, allowed = ["base"]) {
+      async queryChunks(q, k = 6, allowed = ["base"]) {
         if (allowed.length === 0) return [];
         const [qv] = await this.embed([q]);
         const placeholders = allowed.map(() => "?").join(",");
         const res = await this.db.execute({
-          sql: `SELECT c.id, c.text, e.embedding
+          sql: `SELECT c.*, e.embedding
             FROM chunks c
             JOIN embeddings e ON e.id = c.id
             WHERE c.bucket IN (${placeholders})`,
           args: allowed
         });
-        const scored = [];
-        for (const row of res.rows) {
+        const scored = res.rows.map((row) => {
           const emb = JSON.parse(String(row.embedding));
-          const s = _VectorStore.cosine(qv, emb);
-          scored.push({ s, text: String(row.text) });
-        }
-        scored.sort((a, b) => b.s - a.s);
-        return scored.slice(0, k).map((r) => r.text);
+          const trail = row.section == null ? "" : String(row.section);
+          return {
+            text: String(row.text),
+            bucket: String(row.bucket ?? ""),
+            source: String(row.source ?? ""),
+            section: trail ? trail.split(" > ") : [],
+            score: _VectorStore.cosine(qv, emb)
+          };
+        });
+        scored.sort((a, b) => b.score - a.score);
+        return scored.slice(0, k);
+      }
+      /** Same results as {@link queryChunks}, as bare text. */
+      async query(q, k = 6, allowed = ["base"]) {
+        return (await this.queryChunks(q, k, allowed)).map((c) => c.text);
       }
     };
   }
@@ -644,9 +825,10 @@ function detectLeakage(text) {
 function scrubOutput(text) {
   return text.replace(/sk-[A-Za-z0-9]{15,}/g, "[REDACTED]").replace(/AIza[0-9A-Za-z_-]{30,}/g, "[REDACTED]");
 }
-function guardOutput(text) {
+var DEFAULT_REFUSAL = "Sorry, I can't share internal instructions. How else can I help?";
+function guardOutput(text, refusal) {
   if (detectLeakage(text)) {
-    return "Sorry, I can't share internal instructions. How else can I help?";
+    return typeof refusal === "string" && refusal.trim() ? refusal : DEFAULT_REFUSAL;
   }
   return scrubOutput(text);
 }
@@ -658,7 +840,8 @@ async function completeOnce({
   system,
   messages,
   temperature = 0.2,
-  model = DEFAULT_MODEL
+  model = DEFAULT_MODEL,
+  refusal
 }) {
   const res = await client.chat.completions.create({
     model,
@@ -667,7 +850,7 @@ async function completeOnce({
   });
   const text = res.choices[0]?.message?.content ?? "";
   return {
-    content: guardOutput(text),
+    content: guardOutput(text, refusal),
     usage: res.usage || { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }
   };
 }
@@ -692,12 +875,12 @@ async function* completeStream({
 
 // src/core/answer.ts
 var noUsage = () => ({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 });
-async function runAnswerFn(answerFn, { system, messages, mode, sender, conversationId }) {
+async function runAnswerFn(answerFn, { system, messages, mode, sender, conversationId }, refusal) {
   const result = await answerFn({ system, messages, mode, sender, conversationId });
   const raw = typeof result === "string" ? result : result?.content;
   const usage = typeof result === "string" ? void 0 : result?.usage;
   return {
-    content: guardOutput(typeof raw === "string" ? raw : ""),
+    content: guardOutput(typeof raw === "string" ? raw : "", refusal),
     usage: usage ?? noUsage()
   };
 }
@@ -710,12 +893,13 @@ async function answerOnce({
   sender,
   conversationId,
   temperature,
-  model
+  model,
+  refusal
 }) {
   if (answerFn) {
-    return runAnswerFn(answerFn, { system, messages, mode, sender, conversationId });
+    return runAnswerFn(answerFn, { system, messages, mode, sender, conversationId }, refusal);
   }
-  return completeOnce({ client, system, messages, temperature, model });
+  return completeOnce({ client, system, messages, temperature, model, refusal });
 }
 async function applyTransformReply(transformReply, input, logger) {
   if (!transformReply) return input.text;
@@ -739,16 +923,15 @@ async function* answerStream({
   sender,
   conversationId,
   temperature,
-  model
+  model,
+  refusal
 }) {
   if (answerFn) {
-    const { content } = await runAnswerFn(answerFn, {
-      system,
-      messages,
-      mode,
-      sender,
-      conversationId
-    });
+    const { content } = await runAnswerFn(
+      answerFn,
+      { system, messages, mode, sender, conversationId },
+      refusal
+    );
     if (content) yield content;
     return;
   }
@@ -788,6 +971,7 @@ async function resolveBuckets({
 
 // src/index.ts
 init_buildLock();
+init_chunking();
 init_loaders();
 init_logger();
 
@@ -1276,6 +1460,7 @@ async function createMCPServer(config) {
     store = new VectorStore2(createOpenAIEmbedder2(client), {
       databaseClient: db,
       knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
+      chunking: config.chunking,
       logger: log
     });
   }
@@ -1347,7 +1532,8 @@ async function createMCPServer(config) {
           messages: conversationMessages,
           mode: "public",
           conversationId: convId,
-          model
+          model,
+          refusal: config.refusal
         });
         const replyText = await applyTransformReply(
           config.transformReply,
@@ -1437,7 +1623,8 @@ async function createMCPServer(config) {
           messages: conversationMessages,
           mode: "private",
           conversationId: convId,
-          model
+          model,
+          refusal: config.refusal
         });
         const replyText = await applyTransformReply(
           config.transformReply,
@@ -1936,7 +2123,8 @@ function demoRoutes(deps) {
               system,
               messages,
               mode: "public",
-              model
+              model,
+              refusal: config.refusal
             })) {
               await s.write(`data: ${JSON.stringify({ delta })}
 
@@ -1951,7 +2139,8 @@ function demoRoutes(deps) {
           system,
           messages,
           mode: "public",
-          model
+          model,
+          refusal: config.refusal
         });
         const reply = await applyTransformReply(
           config.transformReply,
@@ -2092,7 +2281,8 @@ function openaiRoutes(deps) {
           sender,
           conversationId,
           temperature,
-          model: serverModel
+          model: serverModel,
+          refusal: config.refusal
         })) {
           await s.write(chunk2({ content: delta }, null));
         }
@@ -2109,7 +2299,8 @@ function openaiRoutes(deps) {
       sender,
       conversationId,
       temperature,
-      model: serverModel
+      model: serverModel,
+      refusal: config.refusal
     });
     const reply = await applyTransformReply(
       config.transformReply,
@@ -2200,7 +2391,8 @@ function privateRoutes(deps) {
           messages,
           mode: "private",
           sender,
-          model
+          model,
+          refusal: config.refusal
         })) {
           await s.write(`data: ${JSON.stringify({ delta })}
 
@@ -2216,7 +2408,8 @@ function privateRoutes(deps) {
       messages,
       mode: "private",
       sender,
-      model
+      model,
+      refusal: config.refusal
     });
     const reply = await applyTransformReply(
       config.transformReply,
@@ -2285,7 +2478,8 @@ function publicRoutes(deps) {
           system,
           messages,
           mode: "public",
-          model
+          model,
+          refusal: config.refusal
         })) {
           await s.write(`data: ${JSON.stringify({ delta })}
 
@@ -2294,7 +2488,15 @@ function publicRoutes(deps) {
         await s.write("event: end\ndata: {}\n\n");
       });
     }
-    const out = await answerOnce({ answerFn, client, system, messages, mode: "public", model });
+    const out = await answerOnce({
+      answerFn,
+      client,
+      system,
+      messages,
+      mode: "public",
+      model,
+      refusal: config.refusal
+    });
     const reply = await applyTransformReply(
       config.transformReply,
       { channel: "widget-public", text: out.content },
@@ -2306,7 +2508,7 @@ function publicRoutes(deps) {
 }
 
 // src/server.ts
-import { relative } from "node:path";
+import { relative as relative2 } from "node:path";
 import { Hono as Hono5 } from "hono";
 import OpenAI2 from "openai";
 
@@ -2465,6 +2667,7 @@ async function createServer(config) {
     store = new VectorStore2(createOpenAIEmbedder2(client), {
       databaseClient: db,
       knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
+      chunking: config.chunking,
       logger
     });
   }
@@ -2515,7 +2718,7 @@ async function createServer(config) {
     }
   }
   if (serveStatic && staticDir) {
-    const relativePath = relative(process.cwd(), staticDir);
+    const relativePath = relative2(process.cwd(), staticDir);
     app.get("/chatter.js", serveStatic({ path: `${relativePath}/chatter.js` }));
     app.get("/chatter.css", serveStatic({ path: `${relativePath}/chatter.css` }));
   }
@@ -2562,6 +2765,7 @@ async function createServer(config) {
     app.get("/demo/react", serveStatic({ path: `${publicDir}/react-demo.html` }));
   }
   const senders = createSenderRegistry(logger);
+  const identities = /* @__PURE__ */ new Map();
   const deps = {
     client,
     store,
@@ -2573,6 +2777,7 @@ async function createServer(config) {
     prompts,
     apiKeyManager,
     senders,
+    identities,
     logger
   };
   if (enablePublic) {
@@ -2666,6 +2871,7 @@ export {
   DEFAULT_KNOWLEDGE_DIR,
   DEFAULT_MODEL,
   DEFAULT_PROMPTS_DIR,
+  DEFAULT_REFUSAL,
   PromptLoader,
   VectorStore,
   answerOnce,
@@ -2673,6 +2879,7 @@ export {
   applyTransformReply,
   canAcquireBuildLock,
   chatBodyLimit,
+  chunkSections,
   completeOnce,
   completeStream,
   cors,

package/dist/mcp-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAsBH,YAAY,EACV,WAAW,EACX,QAAQ,EACR,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,gBAAgB;qCA4U3B,IAAI;GAEtC"}
+{"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAsBH,YAAY,EACV,WAAW,EACX,QAAQ,EACR,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,gBAAgB;qCA+U3B,IAAI;GAEtC"}