@pixygon/chatbot-server 0.1.0

package/dist/index.js

@@ -0,0 +1,1169 @@
+// src/models.ts
+function createChatbotModels(mongoose, opts) {
+  const existing = {
+    KnowledgeDocument: mongoose.models.KnowledgeDocument,
+    KnowledgeChunk: mongoose.models.KnowledgeChunk,
+    ChatConversation: mongoose.models.ChatConversation
+  };
+  if (existing.KnowledgeDocument && existing.KnowledgeChunk && existing.ChatConversation) {
+    return existing;
+  }
+  const { Schema } = mongoose;
+  const { tenantField, tenantRefName, plugins = [] } = opts;
+  const docSchema = new Schema(
+    {
+      [tenantField]: { type: Schema.Types.ObjectId, ref: tenantRefName, required: true, index: true },
+      title: { type: String, required: true, trim: true },
+      source: { type: String, default: "" },
+      sourceType: { type: String, enum: ["text", "url", "file"], default: "text" },
+      content: { type: String, required: true },
+      status: { type: String, enum: ["pending", "processing", "ready", "failed"], default: "pending", index: true },
+      chunkCount: { type: Number, default: 0 },
+      processedAt: { type: Date },
+      lastError: { type: String },
+      tags: { type: [String], default: [] },
+      createdBy: { type: Schema.Types.ObjectId, ref: "User" }
+    },
+    { timestamps: true }
+  );
+  docSchema.index({ [tenantField]: 1, status: 1, updatedAt: -1 });
+  for (const plugin of plugins) plugin(docSchema, "KnowledgeDocument");
+  const chunkSchema = new Schema(
+    {
+      [tenantField]: { type: Schema.Types.ObjectId, ref: tenantRefName, required: true, index: true },
+      documentId: { type: Schema.Types.ObjectId, ref: "KnowledgeDocument", required: true, index: true },
+      text: { type: String, required: true },
+      embedding: { type: [Number], required: true },
+      position: { type: Number, required: true },
+      tokens: { type: Number, default: 0 }
+    },
+    { timestamps: true }
+  );
+  chunkSchema.index({ [tenantField]: 1, documentId: 1, position: 1 });
+  for (const plugin of plugins) plugin(chunkSchema, "KnowledgeChunk");
+  const messageSchema = new Schema(
+    {
+      role: { type: String, enum: ["user", "assistant", "system"], required: true },
+      content: { type: String, required: true },
+      citedChunkIds: [{ type: Schema.Types.ObjectId, ref: "KnowledgeChunk" }],
+      tokensInput: { type: Number, default: 0 },
+      tokensOutput: { type: Number, default: 0 },
+      costUsd: { type: Number, default: 0 },
+      model: { type: String },
+      rating: { type: Number, min: -1, max: 1 },
+      retrievalSimilarity: { type: Number, min: 0, max: 1 },
+      retrievedCount: { type: Number, default: 0 },
+      // Cached user-turn embedding — powers semantic clustering without
+      // re-embedding the whole corpus per analytics call. `select:false`
+      // keeps the 1536-float array out of default queries.
+      embedding: { type: [Number], select: false },
+      createdAt: { type: Date, default: Date.now }
+    },
+    { _id: true }
+  );
+  const conversationSchema = new Schema(
+    {
+      [tenantField]: { type: Schema.Types.ObjectId, ref: tenantRefName, required: true, index: true },
+      sessionId: { type: String, required: true, index: true },
+      userId: { type: Schema.Types.ObjectId, ref: "User", index: true, sparse: true },
+      title: { type: String, default: "" },
+      messages: { type: [messageSchema], default: () => [] },
+      totalTokensInput: { type: Number, default: 0 },
+      totalTokensOutput: { type: Number, default: 0 },
+      totalCostUsd: { type: Number, default: 0 },
+      lastMessageAt: { type: Date, default: Date.now, index: true }
+    },
+    { timestamps: true }
+  );
+  conversationSchema.index({ [tenantField]: 1, sessionId: 1, lastMessageAt: -1 });
+  conversationSchema.index({ [tenantField]: 1, lastMessageAt: -1 });
+  for (const plugin of plugins) plugin(conversationSchema, "ChatConversation");
+  return {
+    KnowledgeDocument: mongoose.model("KnowledgeDocument", docSchema),
+    KnowledgeChunk: mongoose.model("KnowledgeChunk", chunkSchema),
+    ChatConversation: mongoose.model("ChatConversation", conversationSchema)
+  };
+}
+
+// src/aiGateway.ts
+var approxTokens = (text) => Math.ceil((text || "").length / 4);
+function createAiClient(cfg) {
+  const PIXYGON_API_URL = cfg.pixygonApiUrl || "https://api.pixygon.com/v1";
+  const OPENAI_API_URL = cfg.openaiApiUrl || "https://api.openai.com/v1";
+  const PIXYGON_AI_URL = `${PIXYGON_API_URL}/ai/api`;
+  const CHAT_IN = cfg.chatInputUsdPer1K ?? 25e-4;
+  const CHAT_OUT = cfg.chatOutputUsdPer1K ?? 0.01;
+  const EMBED_RATE = cfg.embedUsdPer1K ?? 2e-5;
+  return {
+    async chat({ messages, system, model = "openai", version = "gpt-4o" }) {
+      if (!cfg.pixygonApiKey) {
+        const err = new Error("PIXYGON_API_KEY not set");
+        err.code = "PIXYGON_AI_UNCONFIGURED";
+        throw err;
+      }
+      const prompt = messages.map((m) => `${m.role.toUpperCase()}: ${m.content}`).join("\n\n");
+      const formData = new FormData();
+      formData.append("type", "text");
+      formData.append("model", model);
+      formData.append("version", version);
+      formData.append("prompt", prompt);
+      if (system) formData.append("instructions", system);
+      const res = await fetch(PIXYGON_AI_URL, {
+        method: "POST",
+        headers: { "x-api-key": cfg.pixygonApiKey },
+        body: formData
+      });
+      if (!res.ok) {
+        const body = await res.text().catch(() => "");
+        const err = new Error(`Pixygon AI chat failed: ${res.status} \u2014 ${body.slice(0, 300)}`);
+        err.code = "PIXYGON_AI_CHAT_FAILED";
+        err.status = res.status;
+        throw err;
+      }
+      const raw = await res.text();
+      let payload;
+      try {
+        payload = JSON.parse(raw);
+      } catch {
+        payload = { text: raw };
+      }
+      const content = String(payload?.text || payload?.response || payload?.message || raw).trim();
+      const tokensInput = approxTokens(prompt + (system || ""));
+      const tokensOutput = approxTokens(content);
+      const costUsd = tokensInput / 1e3 * CHAT_IN + tokensOutput / 1e3 * CHAT_OUT;
+      return { content, model: `${model}/${version}`, tokensInput, tokensOutput, costUsd };
+    },
+    async embed(text, opts = {}) {
+      if (!cfg.openaiApiKey) {
+        const err = new Error("OPENAI_API_KEY not set");
+        err.code = "OPENAI_UNCONFIGURED";
+        throw err;
+      }
+      const model = opts.model || "text-embedding-3-small";
+      const res = await fetch(`${OPENAI_API_URL}/embeddings`, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${cfg.openaiApiKey}`,
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify({ input: text, model })
+      });
+      if (!res.ok) {
+        const body = await res.text().catch(() => "");
+        const err = new Error(`OpenAI embed failed: ${res.status} \u2014 ${body.slice(0, 300)}`);
+        err.code = "OPENAI_EMBED_FAILED";
+        err.status = res.status;
+        throw err;
+      }
+      const payload = await res.json();
+      const embedding = payload?.data?.[0]?.embedding;
+      if (!Array.isArray(embedding) || embedding.length === 0) {
+        const err = new Error("OpenAI embed response had no vector");
+        err.code = "OPENAI_EMBED_EMPTY";
+        throw err;
+      }
+      const tokens = Number(payload?.usage?.total_tokens ?? approxTokens(text));
+      const costUsd = tokens / 1e3 * EMBED_RATE;
+      return { embedding, tokens, costUsd, dimensions: embedding.length };
+    }
+  };
+}
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length || a.length === 0) return 0;
+  let dot = 0, ma = 0, mb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    ma += a[i] * a[i];
+    mb += b[i] * b[i];
+  }
+  if (ma === 0 || mb === 0) return 0;
+  return dot / (Math.sqrt(ma) * Math.sqrt(mb));
+}
+
+// src/textChunking.ts
+var TARGET_CHARS = 2e3;
+var OVERLAP_CHARS = 400;
+function chunkText(input) {
+  const normalized = (input || "").replace(/\r\n/g, "\n").trim();
+  if (!normalized) return [];
+  const paragraphs = normalized.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+  const chunks = [];
+  let buffer = "";
+  let position = 0;
+  const flush = () => {
+    if (!buffer.trim()) return;
+    chunks.push({ text: buffer.trim(), position: position++ });
+    buffer = buffer.slice(-OVERLAP_CHARS);
+  };
+  for (const p of paragraphs) {
+    if (p.length > TARGET_CHARS) {
+      flush();
+      let cursor = 0;
+      while (cursor < p.length) {
+        chunks.push({ text: p.slice(cursor, cursor + TARGET_CHARS), position: position++ });
+        cursor += TARGET_CHARS - OVERLAP_CHARS;
+      }
+      buffer = "";
+      continue;
+    }
+    if (buffer.length + p.length + 2 > TARGET_CHARS) flush();
+    buffer += (buffer ? "\n\n" : "") + p;
+  }
+  flush();
+  return chunks;
+}
+
+// src/rag.ts
+var TOP_K = 5;
+var MIN_SIMILARITY = 0.3;
+var MAX_HISTORY = 10;
+var defaultIdentity = (_id, fn) => fn();
+function createRag(chatbot) {
+  const { models, ai, config } = chatbot;
+  const { KnowledgeChunk, KnowledgeDocument, ChatConversation } = models;
+  const tenantField = config.tenantField;
+  const withScope = config.hooks.withTenantScope || defaultIdentity;
+  async function respond({ tenantId, sessionId, userId, message }) {
+    if (config.hooks.getCostCap) {
+      const cap = await config.hooks.getCostCap(tenantId);
+      if (cap != null && cap > 0) {
+        const used = await currentMonthCost(tenantId);
+        if (used >= cap) {
+          const err = new Error(`Monthly chat budget reached ($${used.toFixed(4)} of $${cap.toFixed(2)}).`);
+          err.status = 503;
+          err.code = "CHAT_BUDGET_EXCEEDED";
+          throw err;
+        }
+      }
+    }
+    const queryEmbed = await ai.embed(message);
+    const chunks = await KnowledgeChunk.find({ [tenantField]: tenantId }).select("documentId text embedding").lean();
+    const scored = chunks.map((c) => ({ chunk: c, score: cosineSimilarity(queryEmbed.embedding, c.embedding) })).filter(({ score }) => score >= MIN_SIMILARITY).sort((a, b) => b.score - a.score).slice(0, TOP_K);
+    const docIds = Array.from(new Set(scored.map((s) => String(s.chunk.documentId))));
+    const docs = await KnowledgeDocument.find({ _id: { $in: docIds } }).select("title").lean();
+    const titleById = new Map(docs.map((d) => [String(d._id), d.title]));
+    const citations = scored.map(({ chunk, score }) => ({
+      chunkId: chunk._id,
+      documentId: chunk.documentId,
+      documentTitle: titleById.get(String(chunk.documentId)) || "Untitled",
+      snippet: chunk.text.slice(0, 200) + (chunk.text.length > 200 ? "\u2026" : ""),
+      similarity: score
+    }));
+    let conversation = await ChatConversation.findOne({ [tenantField]: tenantId, sessionId });
+    if (!conversation) {
+      conversation = await ChatConversation.create({ [tenantField]: tenantId, sessionId, userId, messages: [] });
+    }
+    const historyMessages = conversation.messages.slice(-MAX_HISTORY).map((m) => ({ role: m.role, content: m.content }));
+    const tenantName = await config.hooks.getTenantName(tenantId) || "the assistant";
+    const system = config.hooks.systemPromptBuilder(tenantName, scored.map(({ chunk }) => chunk.text));
+    const completion = await ai.chat({
+      system,
+      messages: [...historyMessages, { role: "user", content: message }]
+    });
+    conversation.messages.push({
+      role: "user",
+      content: message,
+      tokensInput: queryEmbed.tokens,
+      tokensOutput: 0,
+      costUsd: queryEmbed.costUsd,
+      embedding: queryEmbed.embedding
+    });
+    conversation.messages.push({
+      role: "assistant",
+      content: completion.content,
+      citedChunkIds: citations.map((c) => c.chunkId),
+      tokensInput: completion.tokensInput,
+      tokensOutput: completion.tokensOutput,
+      costUsd: completion.costUsd,
+      model: completion.model,
+      retrievalSimilarity: scored[0]?.score || 0,
+      retrievedCount: scored.length
+    });
+    conversation.totalTokensInput += completion.tokensInput + queryEmbed.tokens;
+    conversation.totalTokensOutput += completion.tokensOutput;
+    conversation.totalCostUsd += completion.costUsd + queryEmbed.costUsd;
+    conversation.lastMessageAt = /* @__PURE__ */ new Date();
+    if (!conversation.title && message) conversation.title = message.slice(0, 80);
+    await conversation.save();
+    return {
+      conversationId: conversation._id,
+      assistantContent: completion.content,
+      citations,
+      costUsd: completion.costUsd + queryEmbed.costUsd
+    };
+  }
+  async function processDocument(documentId) {
+    const doc = await KnowledgeDocument.findById(documentId);
+    if (!doc) return;
+    doc.status = "processing";
+    await doc.save();
+    try {
+      await KnowledgeChunk.deleteMany({ documentId: doc._id, [tenantField]: doc[tenantField] });
+      const chunks = chunkText(doc.content);
+      for (const c of chunks) {
+        const { embedding, tokens } = await ai.embed(c.text);
+        await KnowledgeChunk.create({
+          [tenantField]: doc[tenantField],
+          documentId: doc._id,
+          text: c.text,
+          embedding,
+          position: c.position,
+          tokens
+        });
+      }
+      doc.chunkCount = chunks.length;
+      doc.status = "ready";
+      doc.processedAt = /* @__PURE__ */ new Date();
+      doc.lastError = void 0;
+      await doc.save();
+    } catch (err) {
+      doc.status = "failed";
+      doc.lastError = err?.message || String(err);
+      await doc.save();
+      throw err;
+    }
+  }
+  async function currentMonthCost(tenantId, now = /* @__PURE__ */ new Date()) {
+    const monthStart = new Date(now.getFullYear(), now.getMonth(), 1);
+    const conversations = await ChatConversation.find({
+      [tenantField]: tenantId,
+      lastMessageAt: { $gte: monthStart }
+    }).select("messages").lean();
+    let total = 0;
+    for (const c of conversations) {
+      for (const m of c.messages || []) {
+        const at = m.createdAt instanceof Date ? m.createdAt : new Date(m.createdAt);
+        if (at >= monthStart) total += m.costUsd || 0;
+      }
+    }
+    return total;
+  }
+  return { respond, processDocument, currentMonthCost };
+}
+
+// src/analytics.ts
+var STOPWORDS = /* @__PURE__ */ new Set([
+  // English
+  "the",
+  "a",
+  "an",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "i",
+  "you",
+  "he",
+  "she",
+  "it",
+  "we",
+  "they",
+  "me",
+  "him",
+  "her",
+  "us",
+  "them",
+  "this",
+  "that",
+  "these",
+  "those",
+  "what",
+  "which",
+  "who",
+  "when",
+  "where",
+  "why",
+  "how",
+  "and",
+  "but",
+  "or",
+  "if",
+  "then",
+  "else",
+  "for",
+  "to",
+  "of",
+  "in",
+  "on",
+  "at",
+  "by",
+  "with",
+  "from",
+  "as",
+  "do",
+  "does",
+  "did",
+  "have",
+  "has",
+  "had",
+  "can",
+  "could",
+  "would",
+  "should",
+  "will",
+  "shall",
+  "may",
+  "might",
+  "must",
+  "not",
+  "no",
+  "yes",
+  "my",
+  "your",
+  "his",
+  "their",
+  "our",
+  "its",
+  "so",
+  "than",
+  "too",
+  "very",
+  "just",
+  "any",
+  "some",
+  "all",
+  "also",
+  "more",
+  "most",
+  // Norwegian
+  "og",
+  "i",
+  "jeg",
+  "det",
+  "at",
+  "en",
+  "et",
+  "den",
+  "til",
+  "er",
+  "som",
+  "p\xE5",
+  "de",
+  "med",
+  "han",
+  "av",
+  "ikke",
+  "der",
+  "s\xE5",
+  "var",
+  "meg",
+  "seg",
+  "men",
+  "har",
+  "om",
+  "vi",
+  "ha",
+  "hadde",
+  "hun",
+  "n\xE5",
+  "over",
+  "da",
+  "ved",
+  "fra",
+  "du",
+  "ut",
+  "sin",
+  "dem",
+  "oss",
+  "opp",
+  "man",
+  "kan",
+  "hans",
+  "hvor",
+  "eller",
+  "hva",
+  "skal",
+  "selv",
+  "her",
+  "alle",
+  "vil",
+  "bli",
+  "ble",
+  "kunne",
+  "inn",
+  "n\xE5r",
+  "v\xE6re",
+  "kom",
+  "noen",
+  "noe",
+  "ville",
+  "dere",
+  "kun",
+  "ja",
+  "etter",
+  "ned",
+  "skulle",
+  "denne",
+  "deg",
+  "si",
+  "sine",
+  "sitt",
+  "mot",
+  "\xE5",
+  "hvordan",
+  "ingen",
+  "din",
+  "ditt",
+  "blir",
+  "samme",
+  "b\xE5de",
+  "bare",
+  "enn",
+  "fordi",
+  "f\xF8r",
+  "mange",
+  "ogs\xE5",
+  "slik",
+  // Swedish
+  "och",
+  "\xE4r",
+  "som",
+  "p\xE5",
+  "den",
+  "med",
+  "var",
+  "sig",
+  "f\xF6r",
+  "s\xE5",
+  "till",
+  "har",
+  "de",
+  "att",
+  "ett",
+  "om",
+  "hade",
+  "mot",
+  "under",
+  "n\xE5got",
+  "annat",
+  "s\xE4ga",
+  "ska",
+  "sitt",
+  "hennes"
+]);
+var POOR_SIMILARITY = 0.4;
+var CLUSTER_THRESHOLD = 0.78;
+var CLUSTER_SAMPLE_LIMIT = 5e3;
+function normalize(text) {
+  return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();
+}
+function tokenize(text) {
+  return normalize(text).split(" ").filter((t) => t.length >= 3 && !STOPWORDS.has(t));
+}
+function cosine(a, b) {
+  if (a.length !== b.length || a.length === 0) return 0;
+  let dot = 0, ma = 0, mb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    ma += a[i] * a[i];
+    mb += b[i] * b[i];
+  }
+  if (ma === 0 || mb === 0) return 0;
+  return dot / (Math.sqrt(ma) * Math.sqrt(mb));
+}
+function createAnalytics(chatbot) {
+  const { ChatConversation, KnowledgeChunk, KnowledgeDocument } = chatbot.models;
+  const tenantField = chatbot.config.tenantField;
+  return {
+    async overview(tenantId) {
+      const conversations = await ChatConversation.find({ [tenantField]: tenantId }).select("messages totalCostUsd totalTokensInput totalTokensOutput").lean();
+      let messages = 0, helpful = 0, unhelpful = 0, unrated = 0;
+      let totalCost = 0, totalIn = 0, totalOut = 0;
+      for (const c of conversations) {
+        messages += c.messages?.length || 0;
+        totalCost += c.totalCostUsd || 0;
+        totalIn += c.totalTokensInput || 0;
+        totalOut += c.totalTokensOutput || 0;
+        for (const m of c.messages || []) {
+          if (m.role !== "assistant") continue;
+          if (m.rating === 1) helpful++;
+          else if (m.rating === -1) unhelpful++;
+          else unrated++;
+        }
+      }
+      return {
+        conversations: conversations.length,
+        messages,
+        totalCostUsd: totalCost,
+        totalTokensInput: totalIn,
+        totalTokensOutput: totalOut,
+        ratingsHelpful: helpful,
+        ratingsUnhelpful: unhelpful,
+        ratingsUnrated: unrated
+      };
+    },
+    async topQuestions(tenantId, limit = 20) {
+      const conversations = await ChatConversation.find({ [tenantField]: tenantId }).select("messages").lean();
+      const groups = /* @__PURE__ */ new Map();
+      for (const c of conversations) {
+        for (const m of c.messages || []) {
+          if (m.role !== "user") continue;
+          const norm = normalize(m.content);
+          if (norm.length < 3) continue;
+          const at = m.createdAt instanceof Date ? m.createdAt : new Date(m.createdAt);
+          const existing = groups.get(norm);
+          if (existing) {
+            existing.count++;
+            if (at > existing.lastAt) existing.lastAt = at;
+          } else {
+            groups.set(norm, { question: m.content, count: 1, lastAt: at });
+          }
+        }
+      }
+      return Array.from(groups.entries()).map(([normalized, v]) => ({
+        question: v.question,
+        normalized,
+        count: v.count,
+        lastAskedAt: v.lastAt.toISOString()
+      })).sort((a, b) => b.count - a.count).slice(0, limit);
+    },
+    async keywordFrequency(tenantId, limit = 30) {
+      const conversations = await ChatConversation.find({ [tenantField]: tenantId }).select("messages").lean();
+      const counts = /* @__PURE__ */ new Map();
+      for (const c of conversations) {
+        for (const m of c.messages || []) {
+          if (m.role !== "user") continue;
+          for (const t of tokenize(m.content)) counts.set(t, (counts.get(t) || 0) + 1);
+        }
+      }
+      return Array.from(counts.entries()).map(([term, count]) => ({ term, count })).sort((a, b) => b.count - a.count).slice(0, limit);
+    },
+    async costTimeseries(tenantId, days = 30) {
+      const since = new Date(Date.now() - days * 864e5);
+      const conversations = await ChatConversation.find({
+        [tenantField]: tenantId,
+        lastMessageAt: { $gte: since }
+      }).select("messages").lean();
+      const byDay = /* @__PURE__ */ new Map();
+      for (const c of conversations) {
+        for (const m of c.messages || []) {
+          const at = m.createdAt instanceof Date ? m.createdAt : new Date(m.createdAt);
+          if (at < since) continue;
+          const date = at.toISOString().slice(0, 10);
+          let bucket = byDay.get(date);
+          if (!bucket) {
+            bucket = { date, costUsd: 0, tokensInput: 0, tokensOutput: 0, messages: 0 };
+            byDay.set(date, bucket);
+          }
+          bucket.costUsd += m.costUsd || 0;
+          bucket.tokensInput += m.tokensInput || 0;
+          bucket.tokensOutput += m.tokensOutput || 0;
+          bucket.messages++;
+        }
+      }
+      return Array.from(byDay.values()).sort((a, b) => a.date.localeCompare(b.date));
+    },
+    async knowledgeGaps(tenantId, limit = 15) {
+      const conversations = await ChatConversation.find({ [tenantField]: tenantId }).select("messages").lean();
+      const buckets2 = /* @__PURE__ */ new Map();
+      for (const c of conversations) {
+        const msgs = c.messages || [];
+        for (let i = 0; i < msgs.length; i++) {
+          const m = msgs[i];
+          if (m.role !== "user") continue;
+          const norm = normalize(m.content);
+          if (norm.length < 3) continue;
+          const at = m.createdAt instanceof Date ? m.createdAt : new Date(m.createdAt);
+          let b = buckets2.get(norm);
+          if (!b) {
+            b = { question: m.content, count: 0, similaritySum: 0, similarityCount: 0, negatives: 0, zeroRetrieval: 0, lastAt: at };
+            buckets2.set(norm, b);
+          }
+          b.count++;
+          if (at > b.lastAt) b.lastAt = at;
+          const reply = msgs[i + 1];
+          if (reply?.role === "assistant") {
+            if (typeof reply.retrievalSimilarity === "number") {
+              b.similaritySum += reply.retrievalSimilarity;
+              b.similarityCount++;
+            }
+            if (reply.retrievedCount === 0) b.zeroRetrieval++;
+            if (reply.rating === -1) b.negatives++;
+          }
+        }
+      }
+      const ranked = [];
+      for (const [normalized, b] of buckets2) {
+        const avgSimilarity = b.similarityCount > 0 ? b.similaritySum / b.similarityCount : 0;
+        const gapScore = b.count * (1 - avgSimilarity) + 2 * b.negatives;
+        const isGap = b.negatives > 0 || avgSimilarity < POOR_SIMILARITY || b.zeroRetrieval > 0;
+        if (!isGap) continue;
+        ranked.push({
+          question: b.question,
+          normalized,
+          count: b.count,
+          avgSimilarity,
+          negativeRatings: b.negatives,
+          noRetrievalCount: b.zeroRetrieval,
+          gapScore,
+          lastAskedAt: b.lastAt.toISOString()
+        });
+      }
+      return ranked.sort((a, b) => b.gapScore - a.gapScore).slice(0, limit);
+    },
+    async documentUsage(tenantId) {
+      const [docs, chunks, conversations] = await Promise.all([
+        KnowledgeDocument.find({ [tenantField]: tenantId }).select("_id title chunkCount").lean(),
+        KnowledgeChunk.find({ [tenantField]: tenantId }).select("_id documentId").lean(),
+        ChatConversation.find({ [tenantField]: tenantId }).select("messages").lean()
+      ]);
+      const chunkToDoc = /* @__PURE__ */ new Map();
+      for (const c of chunks) chunkToDoc.set(String(c._id), String(c.documentId));
+      const stats = /* @__PURE__ */ new Map();
+      for (const d of docs) {
+        stats.set(String(d._id), { citations: 0, conversationsWith: /* @__PURE__ */ new Set(), citedChunks: /* @__PURE__ */ new Set() });
+      }
+      for (const conv of conversations) {
+        for (const m of conv.messages || []) {
+          if (m.role !== "assistant" || !m.citedChunkIds?.length) continue;
+          const docsHit = /* @__PURE__ */ new Set();
+          for (const chunkId of m.citedChunkIds) {
+            const docId = chunkToDoc.get(String(chunkId));
+            if (!docId) continue;
+            const s = stats.get(docId);
+            if (!s) continue;
+            s.citations++;
+            s.citedChunks.add(String(chunkId));
+            docsHit.add(docId);
+          }
+          for (const docId of docsHit) stats.get(docId).conversationsWith.add(String(conv._id));
+        }
+      }
+      return docs.map((d) => {
+        const s = stats.get(String(d._id));
+        const status = s.citations === 0 ? "unused" : s.citations < 3 ? "underused" : "active";
+        return {
+          documentId: String(d._id),
+          title: d.title,
+          citationCount: s.citations,
+          uniqueConversations: s.conversationsWith.size,
+          chunkCount: d.chunkCount || 0,
+          citedChunkCount: s.citedChunks.size,
+          status
+        };
+      }).sort((a, b) => b.citationCount - a.citationCount);
+    },
+    async conversationsForQuestion(tenantId, normalized, limit = 50) {
+      const conversations = await ChatConversation.find({ [tenantField]: tenantId }).select("sessionId title messages createdAt").sort({ lastMessageAt: -1 }).lean();
+      const matches = [];
+      for (const c of conversations) {
+        const msgs = c.messages || [];
+        for (let i = 0; i < msgs.length; i++) {
+          const m = msgs[i];
+          if (m.role !== "user") continue;
+          if (normalize(m.content) !== normalized) continue;
+          const reply = msgs[i + 1];
+          matches.push({
+            _id: String(c._id),
+            sessionId: c.sessionId,
+            title: c.title || m.content.slice(0, 80),
+            messageCount: msgs.length,
+            question: m.content,
+            assistantReply: reply?.role === "assistant" ? reply.content : "",
+            rating: reply?.rating,
+            retrievalSimilarity: reply?.retrievalSimilarity,
+            createdAt: (m.createdAt instanceof Date ? m.createdAt : new Date(m.createdAt)).toISOString()
+          });
+          if (matches.length >= limit) return matches;
+          break;
+        }
+      }
+      return matches;
+    },
+    async semanticClusters(tenantId, limit = 15) {
+      const conversations = await ChatConversation.find({ [tenantField]: tenantId }).select("messages").lean();
+      const turns = [];
+      for (const c of conversations) {
+        for (const m of c.messages || []) {
+          if (m.role !== "user") continue;
+          if (!Array.isArray(m.embedding) || m.embedding.length === 0) continue;
+          turns.push({ content: m.content, embedding: m.embedding });
+          if (turns.length >= CLUSTER_SAMPLE_LIMIT) break;
+        }
+        if (turns.length >= CLUSTER_SAMPLE_LIMIT) break;
+      }
+      if (turns.length === 0) return [];
+      const clusters = [];
+      for (const t of turns) {
+        let bestIdx = -1, bestSim = CLUSTER_THRESHOLD;
+        for (let i = 0; i < clusters.length; i++) {
+          const sim = cosine(t.embedding, clusters[i].centroid);
+          if (sim > bestSim) {
+            bestSim = sim;
+            bestIdx = i;
+          }
+        }
+        if (bestIdx === -1) {
+          clusters.push({ centroid: t.embedding.slice(), members: [t], similaritySum: 1 });
+        } else {
+          const c = clusters[bestIdx];
+          const n = c.members.length;
+          for (let i = 0; i < c.centroid.length; i++) {
+            c.centroid[i] = (c.centroid[i] * n + t.embedding[i]) / (n + 1);
+          }
+          c.members.push(t);
+          c.similaritySum += bestSim;
+        }
+      }
+      return clusters.map((c) => {
+        let best = c.members[0], bestSim = -2;
+        for (const m of c.members) {
+          const s = cosine(m.embedding, c.centroid);
+          if (s > bestSim) {
+            bestSim = s;
+            best = m;
+          }
+        }
+        return {
+          representative: best.content,
+          count: c.members.length,
+          members: c.members.slice(0, 5).map((m) => m.content),
+          avgSimilarity: c.similaritySum / c.members.length
+        };
+      }).sort((a, b) => b.count - a.count).slice(0, limit);
+    }
+  };
+}
+
+// src/routes.ts
+import { Router } from "express";
+import crypto from "crypto";
+import { z } from "zod";
+
+// src/rateLimit.ts
+var buckets = /* @__PURE__ */ new Map();
+function rateLimit({
+  windowMs,
+  max,
+  scope
+}) {
+  return (req, res, next) => {
+    const ip = req.headers["x-forwarded-for"]?.split(",")[0]?.trim() || req.socket.remoteAddress || "unknown";
+    const key = `${scope}:${ip}`;
+    const now = Date.now();
+    let bucket = buckets.get(key);
+    if (!bucket || bucket.resetAt < now) {
+      bucket = { count: 0, resetAt: now + windowMs };
+      buckets.set(key, bucket);
+    }
+    bucket.count++;
+    res.setHeader("X-RateLimit-Limit", String(max));
+    res.setHeader("X-RateLimit-Remaining", String(Math.max(0, max - bucket.count)));
+    res.setHeader("X-RateLimit-Reset", String(Math.ceil(bucket.resetAt / 1e3)));
+    if (bucket.count > max) {
+      res.status(429).json({
+        error: "Too many requests, please slow down",
+        code: "RATE_LIMITED",
+        retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1e3)
+      });
+      return;
+    }
+    next();
+  };
+}
+
+// src/routes.ts
+function createPrivateRouter(chatbot) {
+  const rag = createRag(chatbot);
+  const analytics = createAnalytics(chatbot);
+  const { KnowledgeDocument, KnowledgeChunk, ChatConversation } = chatbot.models;
+  const tenantField = chatbot.config.tenantField;
+  const tenantParam = chatbot.config.tenantParamName;
+  const r = Router({ mergeParams: true });
+  const getTenantId = (req) => req.params[tenantParam];
+  const createSchema = z.object({
+    title: z.string().min(1),
+    content: z.string().min(10),
+    source: z.string().optional(),
+    sourceType: z.enum(["text", "url", "file"]).optional(),
+    tags: z.array(z.string()).optional()
+  });
+  r.get("/knowledge", async (req, res, next) => {
+    try {
+      const docs = await KnowledgeDocument.find({ [tenantField]: getTenantId(req) }).select("title source sourceType status chunkCount processedAt lastError tags createdAt updatedAt").sort({ updatedAt: -1 }).lean();
+      res.json(docs);
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.get("/knowledge/:documentId", async (req, res, next) => {
+    try {
+      const doc = await KnowledgeDocument.findById(req.params.documentId).lean();
+      if (!doc) return res.status(404).json({ error: "Not found" });
+      res.json(doc);
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.post("/knowledge", async (req, res, next) => {
+    try {
+      const data = createSchema.parse(req.body);
+      const doc = await KnowledgeDocument.create({
+        ...data,
+        [tenantField]: getTenantId(req),
+        createdBy: req.userId
+      });
+      rag.processDocument(doc._id).catch((e) => console.error(`[chatbot] processDocument ${doc._id} failed:`, e.message));
+      res.status(201).json(doc);
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.put("/knowledge/:documentId", async (req, res, next) => {
+    try {
+      const doc = await KnowledgeDocument.findById(req.params.documentId);
+      if (!doc) return res.status(404).json({ error: "Not found" });
+      const patch = createSchema.partial().parse(req.body);
+      Object.assign(doc, patch);
+      await doc.save();
+      if (patch.content !== void 0) {
+        rag.processDocument(doc._id).catch((e) => console.error(`[chatbot] re-processDocument failed:`, e.message));
+      }
+      res.json(doc);
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.delete("/knowledge/:documentId", async (req, res, next) => {
+    try {
+      const doc = await KnowledgeDocument.findById(req.params.documentId);
+      if (!doc) return res.status(404).json({ error: "Not found" });
+      await KnowledgeChunk.deleteMany({ documentId: doc._id, [tenantField]: doc[tenantField] });
+      await doc.deleteOne();
+      res.json({ deleted: true });
+    } catch (e) {
+      next(e);
+    }
+  });
+  const sendSchema = z.object({
+    sessionId: z.string().min(1).optional(),
+    message: z.string().min(1).max(2e3)
+  });
+  r.post("/chat", async (req, res, next) => {
+    try {
+      const { sessionId: raw, message } = sendSchema.parse(req.body);
+      const sessionId = raw || crypto.randomUUID();
+      const result = await rag.respond({
+        tenantId: getTenantId(req),
+        sessionId,
+        message,
+        userId: req.userId
+      });
+      res.json({
+        sessionId,
+        conversationId: result.conversationId,
+        content: result.assistantContent,
+        citations: result.citations,
+        costUsd: result.costUsd
+      });
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.get("/chat/:sessionId", async (req, res, next) => {
+    try {
+      const c = await ChatConversation.findOne({
+        [tenantField]: getTenantId(req),
+        sessionId: req.params.sessionId
+      }).lean();
+      if (!c) return res.status(404).json({ error: "Not found" });
+      res.json(c);
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.get("/conversations", async (req, res, next) => {
+    try {
+      const limit = Math.max(1, Math.min(200, parseInt(String(req.query.limit) || "50", 10)));
+      const list = await ChatConversation.find({ [tenantField]: getTenantId(req) }).select("sessionId userId title messages totalCostUsd totalTokensInput totalTokensOutput lastMessageAt createdAt").sort({ lastMessageAt: -1 }).limit(limit).lean();
+      res.json(list.map((c) => ({ ...c, messageCount: c.messages?.length || 0, messages: void 0 })));
+    } catch (e) {
+      next(e);
+    }
+  });
+  const rateSchema = z.object({
+    conversationId: z.string().min(1),
+    messageId: z.string().min(1),
+    rating: z.number().int().min(-1).max(1)
+  });
+  r.post("/chat/rate", async (req, res, next) => {
+    try {
+      const { conversationId, messageId, rating } = rateSchema.parse(req.body);
+      const conv = await ChatConversation.findOne({
+        _id: conversationId,
+        [tenantField]: getTenantId(req)
+      });
+      if (!conv) return res.status(404).json({ error: "Conversation not found" });
+      const msg = conv.messages.find((m) => String(m._id) === messageId);
+      if (!msg) return res.status(404).json({ error: "Message not found" });
+      msg.rating = rating;
+      await conv.save();
+      res.json({ rated: true });
+    } catch (e) {
+      next(e);
+    }
+  });
+  const ap = (fn) => async (req, res, next) => {
+    try {
+      await fn(req, res, next);
+    } catch (e) {
+      next(e);
+    }
+  };
+  r.get("/chat-analytics/overview", ap(async (req, res) => res.json(await analytics.overview(getTenantId(req)))));
+  r.get("/chat-analytics/top-questions", ap(async (req, res) => {
+    const lim = Math.max(1, Math.min(100, parseInt(String(req.query.limit) || "20", 10)));
+    res.json(await analytics.topQuestions(getTenantId(req), lim));
+  }));
+  r.get("/chat-analytics/keywords", ap(async (req, res) => {
+    const lim = Math.max(1, Math.min(100, parseInt(String(req.query.limit) || "30", 10)));
+    res.json(await analytics.keywordFrequency(getTenantId(req), lim));
+  }));
+  r.get("/chat-analytics/cost-timeseries", ap(async (req, res) => {
+    const days = Math.max(1, Math.min(180, parseInt(String(req.query.days) || "30", 10)));
+    res.json(await analytics.costTimeseries(getTenantId(req), days));
+  }));
+  r.get("/chat-analytics/knowledge-gaps", ap(async (req, res) => {
+    const lim = Math.max(1, Math.min(100, parseInt(String(req.query.limit) || "15", 10)));
+    res.json(await analytics.knowledgeGaps(getTenantId(req), lim));
+  }));
+  r.get("/chat-analytics/document-usage", ap(async (req, res) => res.json(await analytics.documentUsage(getTenantId(req)))));
+  r.get("/chat-analytics/conversations", ap(async (req, res) => {
+    const normalized = String(req.query.normalized || "").trim();
+    if (!normalized) return res.json([]);
+    const lim = Math.max(1, Math.min(200, parseInt(String(req.query.limit) || "50", 10)));
+    res.json(await analytics.conversationsForQuestion(getTenantId(req), normalized, lim));
+  }));
+  r.get("/chat-analytics/semantic-clusters", ap(async (req, res) => {
+    const lim = Math.max(1, Math.min(50, parseInt(String(req.query.limit) || "15", 10)));
+    res.json(await analytics.semanticClusters(getTenantId(req), lim));
+  }));
+  return r;
+}
+function createPublicRouter(chatbot, opts = {}) {
+  const rag = createRag(chatbot);
+  const tenantField = chatbot.config.tenantField;
+  const { ChatConversation } = chatbot.models;
+  const r = Router();
+  const limiter = rateLimit({
+    windowMs: opts.rateLimitConfig?.windowMs ?? 6e4,
+    max: opts.rateLimitConfig?.max ?? 20,
+    scope: "public-chat"
+  });
+  const sendSchema = z.object({
+    sessionId: z.string().min(1).optional(),
+    message: z.string().min(1).max(2e3)
+  });
+  const rateSchema = z.object({
+    sessionId: z.string().min(1),
+    messageId: z.string().min(1),
+    rating: z.number().int().min(-1).max(1)
+  });
+  r.post("/:tenantSlug", limiter, async (req, res, next) => {
+    try {
+      const slug = String(req.params.tenantSlug || "").toLowerCase();
+      const tenant = await chatbot.config.hooks.getTenantBySlug(slug);
+      if (!tenant) return res.status(404).json({ error: "Tenant not found or inactive" });
+      const { sessionId: raw, message } = sendSchema.parse(req.body);
+      const sessionId = raw || crypto.randomUUID();
+      const withScope = chatbot.config.hooks.withTenantScope || (async (_id, fn) => fn());
+      const result = await withScope(
+        tenant._id,
+        () => rag.respond({ tenantId: tenant._id, sessionId, message })
+      );
+      res.json({
+        sessionId,
+        conversationId: result.conversationId,
+        content: result.assistantContent,
+        citations: result.citations.map((c) => ({
+          documentTitle: c.documentTitle,
+          snippet: c.snippet
+        }))
+      });
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.get("/:tenantSlug/:sessionId", limiter, async (req, res, next) => {
+    try {
+      const slug = String(req.params.tenantSlug || "").toLowerCase();
+      const tenant = await chatbot.config.hooks.getTenantBySlug(slug);
+      if (!tenant) return res.status(404).json({ error: "Tenant not found" });
+      const c = await ChatConversation.findOne({
+        [tenantField]: tenant._id,
+        sessionId: req.params.sessionId
+      }).lean();
+      if (!c) return res.status(404).json({ error: "Conversation not found" });
+      const conversation = c;
+      res.json({
+        sessionId: conversation.sessionId,
+        title: conversation.title,
+        messages: (conversation.messages || []).map((m) => ({
+          _id: m._id,
+          role: m.role,
+          content: m.content,
+          createdAt: m.createdAt,
+          rating: m.rating
+        }))
+      });
+    } catch (e) {
+      next(e);
+    }
+  });
+  r.post("/:tenantSlug/rate", limiter, async (req, res, next) => {
+    try {
+      const slug = String(req.params.tenantSlug || "").toLowerCase();
+      const tenant = await chatbot.config.hooks.getTenantBySlug(slug);
+      if (!tenant) return res.status(404).json({ error: "Tenant not found" });
+      const { sessionId, messageId, rating } = rateSchema.parse(req.body);
+      const conv = await ChatConversation.findOne({
+        [tenantField]: tenant._id,
+        sessionId
+      });
+      if (!conv) return res.status(404).json({ error: "Conversation not found" });
+      const msg = conv.messages.find((m) => String(m._id) === messageId);
+      if (!msg) return res.status(404).json({ error: "Message not found" });
+      msg.rating = rating;
+      await conv.save();
+      res.json({ rated: true });
+    } catch (e) {
+      next(e);
+    }
+  });
+  return r;
+}
+
+// src/index.ts
+function createChatbot(cfg) {
+  const tenantField = cfg.tenantField || cfg.tenantParamName;
+  const models = createChatbotModels(cfg.mongoose, {
+    tenantField,
+    tenantRefName: cfg.tenantRefName,
+    plugins: cfg.plugins
+  });
+  const ai = createAiClient(cfg.ai);
+  const chatbot = {
+    models,
+    ai,
+    config: { ...cfg, tenantField, tenantParamName: cfg.tenantParamName, tenantRefName: cfg.tenantRefName }
+  };
+  const rag = createRag(chatbot);
+  const analytics = createAnalytics(chatbot);
+  const privateRouter = createPrivateRouter(chatbot);
+  const publicRouter = createPublicRouter(chatbot);
+  return {
+    ...chatbot,
+    rag,
+    analytics,
+    routes: { private: privateRouter, public: publicRouter }
+  };
+}
+export {
+  chunkText,
+  cosineSimilarity,
+  createChatbot,
+  rateLimit
+};