@diegoaltoworks/chatter 0.61.0 → 0.62.0

package/dist/mcp-server.mjs

@@ -1,3 +1,393 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res, err) => function __init() {
+  if (err) throw err[0];
+  try {
+    return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+  } catch (e) {
+    throw err = [e], e;
+  }
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/core/logger.ts
+function createConsoleLogger(level = "info") {
+  const threshold = LEVEL_ORDER[level];
+  const write = (at) => (...args) => {
+    if (LEVEL_ORDER[at] < threshold) return;
+    console.error(`[${at}]`, ...args);
+  };
+  return {
+    debug: write("debug"),
+    info: write("info"),
+    warn: write("warn"),
+    error: write("error")
+  };
+}
+function resolveLogger(logger, level) {
+  return logger ?? createConsoleLogger(level);
+}
+var LEVEL_ORDER;
+var init_logger = __esm({
+  "src/core/logger.ts"() {
+    "use strict";
+    LEVEL_ORDER = { debug: 0, info: 1, warn: 2, error: 3 };
+  }
+});
+
+// src/core/buildLock.ts
+function ensureBuildLockSchema(client) {
+  let ready = schemaReady.get(client);
+  if (!ready) {
+    ready = client.execute(
+      `CREATE TABLE IF NOT EXISTS chatter_build_lock (
+          lock_key TEXT PRIMARY KEY,
+          holder_id TEXT NOT NULL,
+          heartbeat_at INTEGER NOT NULL
+        )`
+    ).catch((error) => {
+      schemaReady.delete(client);
+      throw error;
+    });
+    schemaReady.set(client, ready);
+  }
+  return ready;
+}
+function createTursoBuildLock(client) {
+  return {
+    async tryAcquire(key, holderId, now, staleMs) {
+      await ensureBuildLockSchema(client);
+      const result = await client.execute({
+        sql: `INSERT INTO chatter_build_lock (lock_key, holder_id, heartbeat_at) VALUES (?, ?, ?)
+              ON CONFLICT(lock_key) DO UPDATE SET
+                holder_id = excluded.holder_id,
+                heartbeat_at = excluded.heartbeat_at
+              WHERE chatter_build_lock.holder_id = excluded.holder_id
+                 OR chatter_build_lock.heartbeat_at <= excluded.heartbeat_at - ?
+              RETURNING holder_id`,
+        args: [key, holderId, now, staleMs]
+      });
+      const row = result.rows[0];
+      return Boolean(row) && row.holder_id === holderId;
+    },
+    async release(key, holderId) {
+      await ensureBuildLockSchema(client);
+      await client.execute({
+        sql: "DELETE FROM chatter_build_lock WHERE lock_key = ? AND holder_id = ?",
+        args: [key, holderId]
+      });
+    }
+  };
+}
+var BUILD_LOCK_KEY, BUILD_LOCK_STALE_MS, schemaReady;
+var init_buildLock = __esm({
+  "src/core/buildLock.ts"() {
+    "use strict";
+    BUILD_LOCK_KEY = "knowledge_build";
+    BUILD_LOCK_STALE_MS = 10 * 6e4;
+    schemaReady = /* @__PURE__ */ new WeakMap();
+  }
+});
+
+// src/core/loaders.ts
+import { readdirSync, readFileSync as readFileSync2, statSync } from "node:fs";
+import { join as join2 } from "node:path";
+function walk(dir) {
+  const items = [];
+  try {
+    for (const e of readdirSync(dir)) {
+      const p = join2(dir, e);
+      const st = statSync(p);
+      if (st.isDirectory()) items.push(...walk(p));
+      else if (p.endsWith(".md")) items.push(p);
+    }
+  } catch {
+  }
+  return items;
+}
+function loadKnowledge(root = "config/knowledge") {
+  const docs = [];
+  for (const bucket of ["base", "public", "private"]) {
+    const dir = join2(root, bucket);
+    for (const f of walk(dir)) {
+      docs.push({ id: f, text: readFileSync2(f, "utf8"), source: f, bucket });
+    }
+  }
+  return docs;
+}
+var init_loaders = __esm({
+  "src/core/loaders.ts"() {
+    "use strict";
+  }
+});
+
+// src/core/retrieval.ts
+var retrieval_exports = {};
+__export(retrieval_exports, {
+  DATABASE_CONFIG_REQUIRED_MESSAGE: () => DATABASE_CONFIG_REQUIRED_MESSAGE,
+  DEFAULT_KNOWLEDGE_DIR: () => DEFAULT_KNOWLEDGE_DIR,
+  VectorStore: () => VectorStore,
+  createOpenAIEmbedder: () => createOpenAIEmbedder,
+  openLibsqlClient: () => openLibsqlClient,
+  wrapMissingLibsqlError: () => wrapMissingLibsqlError
+});
+function createOpenAIEmbedder(client) {
+  return async (input) => {
+    const res = await client.embeddings.create({ model: EMB_MODEL, input });
+    return res.data.map((d) => d.embedding);
+  };
+}
+function wrapMissingLibsqlError(cause) {
+  return new Error(
+    "Chatter's default knowledge store needs the optional peer dependency '@libsql/client', which is not installed. Install it with `bun add @libsql/client` (or npm/pnpm/yarn), or set config.retriever to use your own retrieval backend instead.",
+    { cause }
+  );
+}
+async function openLibsqlClient(database) {
+  const mod = await import("@libsql/client").catch((error) => {
+    throw wrapMissingLibsqlError(error);
+  });
+  return mod.createClient({ url: database.url, authToken: database.authToken });
+}
+function chunk(text, max = 900) {
+  const out = [];
+  let buf = "";
+  for (const line of text.replace(/\r/g, "").split("\n")) {
+    if (`${buf}
+${line}`.length > max) {
+      out.push(buf.trim());
+      buf = line;
+    } else buf += `
+${line}`;
+  }
+  if (buf.trim()) out.push(buf.trim());
+  return out;
+}
+async function sha256(input) {
+  const data = new TextEncoder().encode(input);
+  const hash = await crypto.subtle.digest("SHA-256", data);
+  return Array.from(new Uint8Array(hash)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+var EMB_MODEL, DEFAULT_KNOWLEDGE_DIR, DATABASE_CONFIG_REQUIRED_MESSAGE, VectorStore;
+var init_retrieval = __esm({
+  "src/core/retrieval.ts"() {
+    "use strict";
+    init_buildLock();
+    init_loaders();
+    init_logger();
+    EMB_MODEL = "text-embedding-3-large";
+    DEFAULT_KNOWLEDGE_DIR = "./config/knowledge";
+    DATABASE_CONFIG_REQUIRED_MESSAGE = "config.database is required unless config.retriever is set - Chatter's default knowledge store (VectorStore) needs a Turso/libsql connection. Set config.database, or supply config.retriever to use your own retrieval backend instead.";
+    VectorStore = class _VectorStore {
+      constructor(embed, options) {
+        this.embed = embed;
+        this.db = options.databaseClient;
+        this.knowledgeDir = options.knowledgeDir || DEFAULT_KNOWLEDGE_DIR;
+        this.logger = options.logger ?? createConsoleLogger();
+        this.buildLock = options.buildLock ?? createTursoBuildLock(this.db);
+        this.instanceId = options.instanceId ?? crypto.randomUUID();
+        this.buildLockStaleMs = options.buildLockStaleMs ?? BUILD_LOCK_STALE_MS;
+      }
+      embed;
+      /**
+       * The libsql client backing this store - the same instance passed in as
+       * `databaseClient`, so callers holding it (e.g. `ServerDependencies.db`)
+       * and the store share one connection.
+       */
+      db;
+      knowledgeDir;
+      logger;
+      buildLock;
+      instanceId;
+      buildLockStaleMs;
+      /**
+       * On boot: ingest new chunks and embed only missing ones.
+       *
+       * The whole ingest runs under a single-writer lock held in the database
+       * (see [buildLock.ts](./buildLock.ts)), because the cleanup step deletes
+       * every chunk id the current `knowledgeDir` did not produce. Without the
+       * lock, a second instance booting mid-build diffs against a database the
+       * first one is still writing and deletes its chunks. An instance that
+       * cannot take the lock skips its build entirely and serves what the holder
+       * has already written, rather than racing it.
+       */
+      async build() {
+        this.logger.info("Building knowledge base...");
+        await this.db.execute(`
+      CREATE TABLE IF NOT EXISTS chunks (id TEXT PRIMARY KEY, bucket TEXT NOT NULL, source TEXT NOT NULL, text TEXT NOT NULL);
+    `);
+        await this.db.execute(`
+      CREATE TABLE IF NOT EXISTS embeddings (id TEXT PRIMARY KEY, model TEXT NOT NULL, embedding BLOB NOT NULL);
+    `);
+        await this.db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_bucket ON chunks(bucket);");
+        if (!await this.renewBuildLock()) {
+          this.logger.warn(
+            "Another instance is building the knowledge base against this database - skipping this build and using what it has written. Nothing was deleted."
+          );
+          return;
+        }
+        try {
+          await this.buildUnderLock();
+        } finally {
+          await this.buildLock.release(BUILD_LOCK_KEY, this.instanceId);
+        }
+      }
+      /** Acquires the build lock, or refreshes the heartbeat if this store already holds it. */
+      renewBuildLock() {
+        return this.buildLock.tryAcquire(
+          BUILD_LOCK_KEY,
+          this.instanceId,
+          Date.now(),
+          this.buildLockStaleMs
+        );
+      }
+      /** The ingest itself. Only ever runs with the build lock held. */
+      async buildUnderLock() {
+        const docs = loadKnowledge(this.knowledgeDir);
+        this.logger.info(`Loaded ${docs.length} knowledge documents`);
+        if (docs.length === 0) {
+          const existing = await this.db.execute("SELECT COUNT(*) as count FROM chunks");
+          const existingCount = Number(existing.rows[0]?.count ?? 0);
+          if (existingCount > 0) {
+            throw new Error(
+              `Refusing to build: loaded 0 knowledge documents from "${this.knowledgeDir}", but ${existingCount} chunks already exist. Proceeding would delete all of them as stale. Check that knowledgeDir resolves to the right path.`
+            );
+          }
+          this.logger.info("No knowledge documents and no existing chunks - nothing to build");
+          return;
+        }
+        const rows = [];
+        for (const d of docs) {
+          for (const part of chunk(d.text)) {
+            const id = await sha256(`${d.bucket}|${d.source}|${part}`);
+            rows.push({ id, bucket: d.bucket, source: d.source, text: part });
+          }
+        }
+        this.logger.info(`Created ${rows.length} chunks from knowledge documents`);
+        await this.cleanupStaleChunks(rows.map((r) => r.id));
+        const UPSERT_BATCH = 500;
+        for (let i = 0; i < rows.length; i += UPSERT_BATCH) {
+          const batch = rows.slice(i, i + UPSERT_BATCH);
+          await this.db.batch(
+            batch.map((r) => ({
+              sql: `INSERT INTO chunks(id,bucket,source,text) VALUES(?,?,?,?)
+                ON CONFLICT(id) DO NOTHING`,
+              args: [r.id, r.bucket, r.source, r.text]
+            })),
+            "write"
+          );
+        }
+        const ids = rows.map((r) => r.id);
+        const missing = [];
+        for (let i = 0; i < ids.length; i += 500) {
+          const batch = ids.slice(i, i + 500);
+          const placeholders = batch.map(() => "?").join(",");
+          const res = await this.db.execute({
+            sql: `SELECT id FROM chunks WHERE id IN (${placeholders})
+              EXCEPT SELECT id FROM embeddings`,
+            args: batch
+          });
+          for (const row of res.rows) missing.push(String(row.id));
+        }
+        if (missing.length === 0) {
+          this.logger.info("No new chunks to embed - knowledge base is up to date");
+          return;
+        }
+        this.logger.info(`Embedding ${missing.length} new/updated chunks with ${EMB_MODEL}...`);
+        const textById = new Map(rows.map((r) => [r.id, r.text]));
+        const BATCH = 96;
+        for (let i = 0; i < missing.length; i += BATCH) {
+          const batchIds = missing.slice(i, i + BATCH);
+          const inputs = batchIds.map((id) => textById.get(id) || "");
+          const vectors = await this.embed(inputs);
+          await this.db.batch(
+            vectors.map((embedding, idx) => ({
+              sql: "INSERT INTO embeddings(id,model,embedding) VALUES(?,?,?)",
+              args: [batchIds[idx], EMB_MODEL, JSON.stringify(embedding)]
+            })),
+            "write"
+          );
+          if (!await this.renewBuildLock()) {
+            this.logger.warn(
+              "Lost the knowledge-base build lock to another instance mid-build - remaining chunks may be embedded by that instance instead."
+            );
+          }
+        }
+        this.logger.info(`Successfully embedded ${missing.length} new chunks`);
+      }
+      // Remove chunks from database that no longer exist in markdown files
+      async cleanupStaleChunks(currentIds) {
+        const result = await this.db.execute("SELECT id FROM chunks");
+        const dbIds = result.rows.map((row) => String(row.id));
+        const currentIdSet = new Set(currentIds);
+        const staleIds = dbIds.filter((id) => !currentIdSet.has(id));
+        if (staleIds.length === 0) {
+          return;
+        }
+        this.logger.info(`Cleaning up ${staleIds.length} stale chunks...`);
+        const BATCH_SIZE = 500;
+        for (let i = 0; i < staleIds.length; i += BATCH_SIZE) {
+          const batch = staleIds.slice(i, i + BATCH_SIZE);
+          const placeholders = batch.map(() => "?").join(",");
+          await this.db.batch(
+            [
+              { sql: `DELETE FROM chunks WHERE id IN (${placeholders})`, args: batch },
+              { sql: `DELETE FROM embeddings WHERE id IN (${placeholders})`, args: batch }
+            ],
+            "write"
+          );
+        }
+        this.logger.info(`Cleaned up ${staleIds.length} stale chunks and their embeddings`);
+      }
+      static cosine(a, b) {
+        let dot = 0;
+        let na = 0;
+        let nb = 0;
+        for (let i = 0; i < a.length; i++) {
+          dot += a[i] * b[i];
+          na += a[i] * a[i];
+          nb += b[i] * b[i];
+        }
+        return dot / (Math.sqrt(na) * Math.sqrt(nb));
+      }
+      /**
+       * Retrieve top-k across allowed buckets; compute similarity in app (simple
+       * & portable).
+       *
+       * `allowed` is a bucket-name list rather than the {@link Bucket} union: the
+       * `chunks` table constrains nothing, so a deployment that ingests its own
+       * buckets can query them (`build` only writes and prunes the three the
+       * knowledge loader walks). Names are bound as query parameters, never
+       * interpolated. An empty list retrieves nothing, and short-circuits before
+       * the embedding call.
+       */
+      async query(q, k = 6, allowed = ["base"]) {
+        if (allowed.length === 0) return [];
+        const [qv] = await this.embed([q]);
+        const placeholders = allowed.map(() => "?").join(",");
+        const res = await this.db.execute({
+          sql: `SELECT c.id, c.text, e.embedding
+            FROM chunks c
+            JOIN embeddings e ON e.id = c.id
+            WHERE c.bucket IN (${placeholders})`,
+          args: allowed
+        });
+        const scored = [];
+        for (const row of res.rows) {
+          const emb = JSON.parse(String(row.embedding));
+          const s = _VectorStore.cosine(qv, emb);
+          scored.push({ s, text: String(row.text) });
+        }
+        scored.sort((a, b) => b.s - a.s);
+        return scored.slice(0, k).map((r) => r.text);
+      }
+    };
+  }
+});
+
 // src/mcp-server.ts
 import OpenAI from "openai";
 
@@ -73,7 +463,8 @@ async function answerOnce({
 async function applyTransformReply(transformReply, input, logger) {
   if (!transformReply) return input.text;
   try {
-    return await transformReply(input);
+    const result = await transformReply(input);
+    return typeof result === "string" ? result : null;
   } catch (error) {
     logger?.error(
       `transformReply threw for channel "${input.channel}"; sending the original reply`,
@@ -114,24 +505,8 @@ async function resolveBuckets({
   return cleaned.filter((bucket) => ceiling.has(bucket));
 }
 
-// src/core/logger.ts
-var LEVEL_ORDER = { debug: 0, info: 1, warn: 2, error: 3 };
-function createConsoleLogger(level = "info") {
-  const threshold = LEVEL_ORDER[level];
-  const write = (at) => (...args) => {
-    if (LEVEL_ORDER[at] < threshold) return;
-    console.error(`[${at}]`, ...args);
-  };
-  return {
-    debug: write("debug"),
-    info: write("info"),
-    warn: write("warn"),
-    error: write("error")
-  };
-}
-function resolveLogger(logger, level) {
-  return logger ?? createConsoleLogger(level);
-}
+// src/mcp-server.ts
+init_logger();
 
 // src/core/messages.ts
 function lastUserMessage(messages) {
@@ -154,6 +529,7 @@ async function prepareChat({
   sender,
   rewriteQuery,
   rerankContext,
+  fallbackFn,
   logger
 }) {
   const lastUserMsg = lastUserMessage(messages);
@@ -170,21 +546,39 @@ async function prepareChat({
       logger?.error("rewriteQuery threw; falling back to the original query", error);
     }
   }
-  let ctx = await store.query(query, topK, buckets ?? defaultBuckets(mode));
+  const resolvedBuckets = buckets ?? defaultBuckets(mode);
+  const retrievedChunks = await store.query(query, topK, resolvedBuckets);
+  let ctx = retrievedChunks;
   if (rerankContext) {
     try {
-      const reranked = await rerankContext({ query, chunks: ctx });
+      const reranked = await rerankContext({ query, chunks: [...ctx] });
       if (Array.isArray(reranked) && reranked.every((c) => typeof c === "string")) ctx = reranked;
     } catch (error) {
       logger?.error("rerankContext threw; falling back to the original chunks", error);
     }
   }
+  let fallback;
+  if (ctx.length === 0 && fallbackFn) {
+    try {
+      const guidance = await fallbackFn({
+        query,
+        mode,
+        buckets: resolvedBuckets,
+        sender,
+        retrievedChunks
+      });
+      if (typeof guidance === "string" && guidance.trim()) fallback = guidance.trim();
+    } catch (error) {
+      logger?.error("fallbackFn threw; proceeding without fallback guidance", error);
+    }
+  }
   const persona = personaLayer?.trim() || (mode === "private" ? prompts.privatePersona : prompts.publicPersona);
   const hint = channelHint?.trim();
   const system = [
     prompts.baseSystemRules,
     persona,
     ...hint ? [hint] : [],
+    ...fallback ? [fallback] : [],
     `${label}:
 ${ctx.join("\n\n")}`
   ].join("\n\n");
@@ -194,6 +588,7 @@ ${ctx.join("\n\n")}`
 // src/core/prompts.ts
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
+var DEFAULT_PROMPTS_DIR = "./config/prompts";
 var PromptLoader = class {
   constructor(promptsDir, bot) {
     this.promptsDir = promptsDir;
@@ -224,206 +619,8 @@ var PromptLoader = class {
   }
 };
 
-// src/core/retrieval.ts
-import { createClient } from "@libsql/client";
-
-// src/core/loaders.ts
-import { readdirSync, readFileSync as readFileSync2, statSync } from "node:fs";
-import { join as join2 } from "node:path";
-function walk(dir) {
-  const items = [];
-  try {
-    for (const e of readdirSync(dir)) {
-      const p = join2(dir, e);
-      const st = statSync(p);
-      if (st.isDirectory()) items.push(...walk(p));
-      else if (p.endsWith(".md")) items.push(p);
-    }
-  } catch {
-  }
-  return items;
-}
-function loadKnowledge(root = "config/knowledge") {
-  const docs = [];
-  for (const bucket of ["base", "public", "private"]) {
-    const dir = join2(root, bucket);
-    for (const f of walk(dir)) {
-      docs.push({ id: f, text: readFileSync2(f, "utf8"), source: f, bucket });
-    }
-  }
-  return docs;
-}
-
-// src/core/retrieval.ts
-var EMB_MODEL = "text-embedding-3-large";
-function chunk(text, max = 900) {
-  const out = [];
-  let buf = "";
-  for (const line of text.replace(/\r/g, "").split("\n")) {
-    if (`${buf}
-${line}`.length > max) {
-      out.push(buf.trim());
-      buf = line;
-    } else buf += `
-${line}`;
-  }
-  if (buf.trim()) out.push(buf.trim());
-  return out;
-}
-async function sha256(input) {
-  const data = new TextEncoder().encode(input);
-  const hash = await crypto.subtle.digest("SHA-256", data);
-  return Array.from(new Uint8Array(hash)).map((b) => b.toString(16).padStart(2, "0")).join("");
-}
-var VectorStore = class _VectorStore {
-  constructor(client, options) {
-    this.client = client;
-    this.db = options.databaseClient ? options.databaseClient : createClient({
-      url: options.databaseUrl,
-      authToken: options.databaseAuthToken
-    });
-    this.knowledgeDir = options.knowledgeDir || "./config/knowledge";
-    this.logger = options.logger ?? createConsoleLogger();
-  }
-  client;
-  /**
-   * The libsql client backing this store. When a client was injected this is
-   * that same instance, so callers holding it (e.g. `ServerDependencies.db`)
-   * and the store share one connection.
-   */
-  db;
-  knowledgeDir;
-  logger;
-  // On boot: ingest new chunks and embed only missing ones.
-  async build() {
-    this.logger.info("\u{1F504} Building knowledge base...");
-    await this.db.execute(`
-      CREATE TABLE IF NOT EXISTS chunks (id TEXT PRIMARY KEY, bucket TEXT NOT NULL, source TEXT NOT NULL, text TEXT NOT NULL);
-    `);
-    await this.db.execute(`
-      CREATE TABLE IF NOT EXISTS embeddings (id TEXT PRIMARY KEY, model TEXT NOT NULL, embedding BLOB NOT NULL);
-    `);
-    await this.db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_bucket ON chunks(bucket);");
-    const docs = loadKnowledge(this.knowledgeDir);
-    this.logger.info(`\u{1F4DA} Loaded ${docs.length} knowledge documents`);
-    const rows = [];
-    for (const d of docs) {
-      for (const part of chunk(d.text)) {
-        const id = await sha256(`${d.bucket}|${d.source}|${part}`);
-        rows.push({ id, bucket: d.bucket, source: d.source, text: part });
-      }
-    }
-    this.logger.info(`\u{1F4E6} Created ${rows.length} chunks from knowledge documents`);
-    await this.cleanupStaleChunks(rows.map((r) => r.id));
-    const UPSERT_BATCH = 500;
-    for (let i = 0; i < rows.length; i += UPSERT_BATCH) {
-      const batch = rows.slice(i, i + UPSERT_BATCH);
-      await this.db.batch(
-        batch.map((r) => ({
-          sql: `INSERT INTO chunks(id,bucket,source,text) VALUES(?,?,?,?)
-                ON CONFLICT(id) DO NOTHING`,
-          args: [r.id, r.bucket, r.source, r.text]
-        })),
-        "write"
-      );
-    }
-    const ids = rows.map((r) => r.id);
-    const missing = [];
-    for (let i = 0; i < ids.length; i += 500) {
-      const batch = ids.slice(i, i + 500);
-      const placeholders = batch.map(() => "?").join(",");
-      const res = await this.db.execute({
-        sql: `SELECT id FROM chunks WHERE id IN (${placeholders})
-              EXCEPT SELECT id FROM embeddings`,
-        args: batch
-      });
-      for (const row of res.rows) missing.push(String(row.id));
-    }
-    if (missing.length === 0) {
-      this.logger.info("\u2705 No new chunks to embed - knowledge base is up to date");
-      return;
-    }
-    this.logger.info(`\u{1F52E} Embedding ${missing.length} new/updated chunks with ${EMB_MODEL}...`);
-    const textById = new Map(rows.map((r) => [r.id, r.text]));
-    const BATCH = 96;
-    for (let i = 0; i < missing.length; i += BATCH) {
-      const batchIds = missing.slice(i, i + BATCH);
-      const inputs = batchIds.map((id) => textById.get(id) || "");
-      const emb = await this.client.embeddings.create({ model: EMB_MODEL, input: inputs });
-      await this.db.batch(
-        emb.data.map((d, idx) => ({
-          sql: "INSERT INTO embeddings(id,model,embedding) VALUES(?,?,?)",
-          args: [batchIds[idx], EMB_MODEL, JSON.stringify(d.embedding)]
-        })),
-        "write"
-      );
-    }
-    this.logger.info(`\u2705 Successfully embedded ${missing.length} new chunks`);
-  }
-  // Remove chunks from database that no longer exist in markdown files
-  async cleanupStaleChunks(currentIds) {
-    const result = await this.db.execute("SELECT id FROM chunks");
-    const dbIds = result.rows.map((row) => String(row.id));
-    const currentIdSet = new Set(currentIds);
-    const staleIds = dbIds.filter((id) => !currentIdSet.has(id));
-    if (staleIds.length === 0) {
-      return;
-    }
-    this.logger.info(`\u{1F9F9} Cleaning up ${staleIds.length} stale chunks...`);
-    const BATCH_SIZE = 500;
-    for (let i = 0; i < staleIds.length; i += BATCH_SIZE) {
-      const batch = staleIds.slice(i, i + BATCH_SIZE);
-      const placeholders = batch.map(() => "?").join(",");
-      await this.db.execute({
-        sql: `DELETE FROM chunks WHERE id IN (${placeholders})`,
-        args: batch
-      });
-    }
-    this.logger.info(`\u2713 Cleaned up ${staleIds.length} stale chunks and their embeddings`);
-  }
-  static cosine(a, b) {
-    let dot = 0;
-    let na = 0;
-    let nb = 0;
-    for (let i = 0; i < a.length; i++) {
-      dot += a[i] * b[i];
-      na += a[i] * a[i];
-      nb += b[i] * b[i];
-    }
-    return dot / (Math.sqrt(na) * Math.sqrt(nb));
-  }
-  /**
-   * Retrieve top-k across allowed buckets; compute similarity in app (simple
-   * & portable).
-   *
-   * `allowed` is a bucket-name list rather than the {@link Bucket} union: the
-   * `chunks` table constrains nothing, so a deployment that ingests its own
-   * buckets can query them (`build` only writes and prunes the three the
-   * knowledge loader walks). Names are bound as query parameters, never
-   * interpolated. An empty list retrieves nothing, and short-circuits before
-   * the embedding call.
-   */
-  async query(q, k = 6, allowed = ["base"]) {
-    if (allowed.length === 0) return [];
-    const qv = (await this.client.embeddings.create({ model: EMB_MODEL, input: [q] })).data[0].embedding;
-    const placeholders = allowed.map(() => "?").join(",");
-    const res = await this.db.execute({
-      sql: `SELECT c.id, c.text, e.embedding
-            FROM chunks c
-            JOIN embeddings e ON e.id = c.id
-            WHERE c.bucket IN (${placeholders})`,
-      args: allowed
-    });
-    const scored = [];
-    for (const row of res.rows) {
-      const emb = JSON.parse(String(row.embedding));
-      const s = _VectorStore.cosine(qv, emb);
-      scored.push({ s, text: String(row.text) });
-    }
-    scored.sort((a, b) => b.s - a.s);
-    return scored.slice(0, k).map((r) => r.text);
-  }
-};
+// src/mcp-server.ts
+init_retrieval();
 
 // src/mcp-server/conversation-id.ts
 function generateConversationId() {
@@ -450,28 +647,38 @@ function calculateCost(usage, rates) {
 }
 
 // src/mcp-server/logger.ts
+init_logger();
 var MCPLogger = class {
   enableConsole;
   callback;
   logger;
-  constructor(enableConsole = true, callback, logger) {
+  logContent;
+  constructor(enableConsole = true, callback, logger, logContent = false) {
     this.enableConsole = enableConsole;
     this.callback = callback;
     this.logger = logger ?? createConsoleLogger();
+    this.logContent = logContent;
   }
   /**
    * Log a chat interaction
    */
   async logChatInteraction(toolName, conversationId, conversationMessages, ragContext, response, duration, cost) {
     const lastUserMsg = lastUserMessage(conversationMessages);
+    const content = {
+      userMessage: scrubOutput(lastUserMsg?.content || ""),
+      conversationHistory: conversationMessages.map((msg) => ({
+        ...msg,
+        content: scrubOutput(msg.content)
+      })),
+      ragContext: ragContext.map(scrubOutput),
+      response: scrubOutput(response)
+    };
+    const timestamp = (/* @__PURE__ */ new Date()).toISOString();
     const logEvent = {
-      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      timestamp,
       toolName,
       conversationId,
-      userMessage: lastUserMsg?.content || "",
-      conversationHistory: conversationMessages,
-      ragContext,
-      response,
+      ...content,
       duration,
       cost
     };
@@ -479,9 +686,27 @@ var MCPLogger = class {
       this.logger.info(
         JSON.stringify({
           event: "mcp_chat",
-          ...logEvent
+          timestamp,
+          toolName,
+          conversationId,
+          messageCount: conversationMessages.length,
+          ragContextCount: ragContext.length,
+          responseChars: response.length,
+          duration,
+          cost
         })
       );
+      if (this.logContent) {
+        this.logger.debug(
+          JSON.stringify({
+            event: "mcp_chat_content",
+            timestamp,
+            toolName,
+            conversationId,
+            ...content
+          })
+        );
+      }
     }
     if (this.callback) {
       try {
@@ -492,11 +717,12 @@ var MCPLogger = class {
     }
   }
 };
-function createLogger(enableConsole = true, callback, logger) {
-  return new MCPLogger(enableConsole, callback, logger);
+function createLogger(enableConsole = true, callback, logger, logContent = false) {
+  return new MCPLogger(enableConsole, callback, logger, logContent);
 }
 
 // src/mcp-server/rate-limiter.ts
+var DEFAULT_WINDOW_MS = 6e4;
 var RateLimiter = class {
   requestMap;
   limit;
@@ -505,9 +731,9 @@ var RateLimiter = class {
    * Create a new rate limiter
    *
    * @param limit - Maximum requests allowed in the time window
-   * @param windowMs - Time window in milliseconds (default: 60000 = 1 minute)
+   * @param windowMs - Time window in milliseconds (default: `DEFAULT_WINDOW_MS`)
    */
-  constructor(limit, windowMs = 6e4) {
+  constructor(limit, windowMs = DEFAULT_WINDOW_MS) {
     this.requestMap = /* @__PURE__ */ new Map();
     this.limit = limit;
     this.windowMs = windowMs;
@@ -571,7 +797,7 @@ var RateLimiter = class {
     return Math.max(0, timeUntilReset);
   }
 };
-function createRateLimiter(limit, windowMs = 6e4) {
+function createMcpRateLimiter(limit, windowMs = DEFAULT_WINDOW_MS) {
   return limit ? new RateLimiter(limit, windowMs) : null;
 }
 
@@ -621,11 +847,15 @@ async function createMCPServer(config) {
   const log = resolveLogger(config.logger, config.logLevel);
   log.info(`\u{1F680} Starting ${config.bot.name} MCP Server...`);
   const [McpServer, z] = await Promise.all([loadMcpSdk(), loadZod()]);
-  const knowledgeDir = config.knowledgeDir || "./config/knowledge";
-  const promptsDir = config.promptsDir || "./config/prompts";
+  const promptsDir = config.promptsDir || DEFAULT_PROMPTS_DIR;
   const transportMode = config.transport || "stdio";
-  const rateLimiter = createRateLimiter(config.toolRateLimit);
-  const logger = createLogger(config.logging?.console !== false, config.logging?.onChat, log);
+  const rateLimiter = createMcpRateLimiter(config.toolRateLimit);
+  const logger = createLogger(
+    config.logging?.console !== false,
+    config.logging?.onChat,
+    log,
+    config.logging?.content === true
+  );
   const publicToolConfig = {
     enabled: config.tools?.public?.enabled !== false,
     name: config.tools?.public?.name || "chat_public",
@@ -640,13 +870,22 @@ async function createMCPServer(config) {
     apiKey: config.openai.apiKey
   });
   const model = config.openai.model || DEFAULT_MODEL;
-  const store = new VectorStore(client, {
-    databaseUrl: config.database.url,
-    databaseAuthToken: config.database.authToken,
-    knowledgeDir,
-    logger: log
-  });
-  await store.build();
+  let store;
+  if (config.retriever) {
+    store = config.retriever;
+  } else {
+    if (!config.database) {
+      throw new Error(DATABASE_CONFIG_REQUIRED_MESSAGE);
+    }
+    const { DEFAULT_KNOWLEDGE_DIR: DEFAULT_KNOWLEDGE_DIR2, VectorStore: VectorStore2, createOpenAIEmbedder: createOpenAIEmbedder2, openLibsqlClient: openLibsqlClient2 } = await Promise.resolve().then(() => (init_retrieval(), retrieval_exports));
+    const db = await openLibsqlClient2(config.database);
+    store = new VectorStore2(createOpenAIEmbedder2(client), {
+      databaseClient: db,
+      knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
+      logger: log
+    });
+  }
+  await store.build?.();
   log.info("\u2705 Knowledge base ready");
   const prompts = new PromptLoader(promptsDir, config.bot);
   const server = new McpServer({
@@ -695,6 +934,7 @@ async function createMCPServer(config) {
           buckets,
           rewriteQuery: config.rewriteQuery,
           rerankContext: config.rerankContext,
+          fallbackFn: config.fallbackFn,
           logger: log
         });
         const result = await answerOnce({
@@ -784,6 +1024,7 @@ async function createMCPServer(config) {
           buckets,
           rewriteQuery: config.rewriteQuery,
           rerankContext: config.rerankContext,
+          fallbackFn: config.fallbackFn,
           logger: log
         });
         const result = await answerOnce({