@chatman-media/kb 1.3.0

package/src/semantic-cache.ts

@@ -0,0 +1,154 @@
+import type { AnswerResult, AnswerTelemetry } from "./answer-types.ts";
+import type { EmbeddingClient } from "@chatman-media/llm-router";
+
+export interface SemanticCacheOptions {
+  /**
+   * Cosine similarity threshold for a cache hit (0–1). Higher = stricter.
+   * Default: 0.92 — catches paraphrases and minor reformulations.
+   */
+  threshold?: number;
+  /**
+   * Maximum number of entries to keep in memory. Oldest entries are evicted
+   * when the limit is reached. Default: 500.
+   */
+  maxEntries?: number;
+  /**
+   * TTL in milliseconds. Entries older than this are ignored. Default: 1 hour.
+   * Set to `Infinity` to disable expiry.
+   */
+  ttlMs?: number;
+}
+
+interface CacheEntry {
+  embedding: number[];
+  result: AnswerResult;
+  createdAt: number;
+}
+
+/**
+ * In-memory semantic cache for `answerWithRag` results.
+ *
+ * Stores (question embedding → AnswerResult) pairs. On lookup, computes
+ * cosine similarity between the incoming question embedding and all cached
+ * embeddings, returning the best match above `threshold`.
+ *
+ * Use `SemanticCache.wrap()` to get a drop-in cached version of any async
+ * function that accepts an `EmbeddingClient` and a question string.
+ *
+ * @example
+ * ```ts
+ * import { SemanticCache, answerWithRag } from "@chatman-media/kb";
+ *
+ * const cache = new SemanticCache(embedder, { threshold: 0.93, ttlMs: 30 * 60_000 });
+ *
+ * const result = await cache.getOrSet(
+ *   input.question,
+ *   () => answerWithRag(input),
+ * );
+ * ```
+ */
+export class SemanticCache {
+  private readonly embedder: EmbeddingClient;
+  private readonly threshold: number;
+  private readonly maxEntries: number;
+  private readonly ttlMs: number;
+  private entries: CacheEntry[] = [];
+
+  /** Total cache hits since creation. */
+  hits = 0;
+  /** Total cache misses since creation. */
+  misses = 0;
+
+  constructor(embedder: EmbeddingClient, opts: SemanticCacheOptions = {}) {
+    this.embedder = embedder;
+    this.threshold = opts.threshold ?? 0.92;
+    this.maxEntries = opts.maxEntries ?? 500;
+    this.ttlMs = opts.ttlMs ?? 60 * 60_000;
+  }
+
+  /**
+   * Look up `question` in the cache. If a similar question was cached and is
+   * still fresh, returns the cached `AnswerResult` with `telemetry.path`
+   * overridden to `"cache_hit"`. Otherwise calls `fn()`, stores the result,
+   * and returns it.
+   */
+  async getOrSet(question: string, fn: () => Promise<AnswerResult>): Promise<AnswerResult> {
+    const [queryVec] = await this.embedder.embed([question]);
+    if (!queryVec) return fn();
+
+    this.evictExpired();
+
+    const hit = this.findBestMatch(queryVec);
+    if (hit) {
+      this.hits++;
+      return {
+        ...hit.result,
+        telemetry: { ...hit.result.telemetry, path: "cache_hit" as AnswerTelemetry["path"] },
+      };
+    }
+
+    this.misses++;
+    const result = await fn();
+    this.store(queryVec, result);
+    return result;
+  }
+
+  /** Manually insert a question→result pair (e.g. from a warm-up script). */
+  async prime(question: string, result: AnswerResult): Promise<void> {
+    const [vec] = await this.embedder.embed([question]);
+    if (vec) this.store(vec, result);
+  }
+
+  /** Remove all entries. */
+  clear(): void {
+    this.entries = [];
+  }
+
+  /** Number of live (non-expired) entries. */
+  get size(): number {
+    this.evictExpired();
+    return this.entries.length;
+  }
+
+  // ── Private ───────────────────────────────────────────────────────────────
+
+  private findBestMatch(queryVec: number[]): CacheEntry | null {
+    let bestSim = -1;
+    let bestEntry: CacheEntry | null = null;
+    for (const entry of this.entries) {
+      const sim = cosineSimilarity(queryVec, entry.embedding);
+      if (sim > bestSim) {
+        bestSim = sim;
+        bestEntry = entry;
+      }
+    }
+    return bestSim >= this.threshold ? bestEntry : null;
+  }
+
+  private store(embedding: number[], result: AnswerResult): void {
+    if (this.entries.length >= this.maxEntries) {
+      // Evict oldest entry
+      this.entries.shift();
+    }
+    this.entries.push({ embedding, result, createdAt: Date.now() });
+  }
+
+  private evictExpired(): void {
+    if (this.ttlMs === Infinity) return;
+    const cutoff = Date.now() - this.ttlMs;
+    this.entries = this.entries.filter((e) => e.createdAt >= cutoff);
+  }
+}
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += (a[i] ?? 0) * (b[i] ?? 0);
+    normA += (a[i] ?? 0) ** 2;
+    normB += (b[i] ?? 0) ** 2;
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/src/server.ts

@@ -0,0 +1,164 @@
+import { answerWithRagStream } from "./answer.ts";
+import type { AnswerInput, AnswerTelemetry } from "./answer-types.ts";
+import type { ChatClient } from "@chatman-media/llm-router";
+import type { EmbeddingClient } from "@chatman-media/llm-router";
+import type { IKbStore } from "./types.ts";
+
+export interface RagServerOptions {
+  /** Knowledge base store. */
+  kb: IKbStore;
+  /** LLM chat client. */
+  chat: ChatClient;
+  /** Embedding client. */
+  embedder: EmbeddingClient;
+  /** Port to listen on. Default: 3000. */
+  port?: number;
+  /** Hostname to bind to. Default: "0.0.0.0". */
+  hostname?: string;
+  /**
+   * Path that the server listens on. Default: "/chat".
+   * POST JSON `{ question, userId?, conversationId?, ... }` → SSE stream of tokens.
+   */
+  path?: string;
+  /**
+   * Extra `AnswerInput` defaults merged into every request.
+   * Request body fields take precedence over these defaults.
+   */
+  defaults?: Partial<Omit<AnswerInput, "question" | "kb" | "chat" | "embedder">>;
+  /**
+   * Called after every answered request with the final telemetry.
+   * Use this to log to your analytics backend.
+   */
+  onTelemetry?: (telemetry: AnswerTelemetry) => void;
+  /**
+   * CORS origin header value. Set to `"*"` to allow all origins.
+   * Default: `undefined` (no CORS headers added).
+   */
+  cors?: string;
+}
+
+/** Shape of the JSON request body accepted by the RAG server. */
+export interface RagRequestBody {
+  question: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Lightweight Bun HTTP server that exposes `answerWithRagStream` as an SSE endpoint.
+ *
+ * **Request** — `POST {path}` with `Content-Type: application/json`:
+ * ```json
+ * { "question": "What is the onboarding process?" }
+ * ```
+ * Any additional fields in the body are merged into `AnswerInput` (e.g. `style`,
+ * `history`, `conversationSummary`, `userFacts`).
+ *
+ * **Response** — `text/event-stream`:
+ * ```
+ * data: Hello
+ * data:  world
+ * data: [DONE]
+ * ```
+ * Each `data:` line carries one streamed token. The stream ends with `data: [DONE]`.
+ * On error the server emits `data: [ERROR] <message>` and closes the stream.
+ *
+ * @example
+ * ```ts
+ * import { createRagServer } from "@chatman-media/kb";
+ *
+ * const server = createRagServer({
+ *   kb, chat, embedder,
+ *   port: 3000,
+ *   cors: "*",
+ *   onTelemetry: (t) => console.log("path:", t.path, "ms:", t.latencyMs),
+ * });
+ *
+ * console.log(`Listening on http://localhost:${server.port}`);
+ * // server.stop() to shut down
+ * ```
+ */
+export function createRagServer(opts: RagServerOptions): ReturnType<typeof Bun.serve> {
+  const path = opts.path ?? "/chat";
+  const corsOrigin = opts.cors;
+
+  function corsHeaders(): Record<string, string> {
+    if (!corsOrigin) return {};
+    return {
+      "access-control-allow-origin": corsOrigin,
+      "access-control-allow-methods": "POST, OPTIONS",
+      "access-control-allow-headers": "content-type",
+    };
+  }
+
+  return Bun.serve({
+    port: opts.port ?? 3000,
+    hostname: opts.hostname ?? "0.0.0.0",
+
+    async fetch(req) {
+      const url = new URL(req.url);
+
+      // CORS preflight
+      if (req.method === "OPTIONS") {
+        return new Response(null, { status: 204, headers: corsHeaders() });
+      }
+
+      if (req.method !== "POST" || url.pathname !== path) {
+        return new Response("Not Found", { status: 404 });
+      }
+
+      let body: RagRequestBody;
+      try {
+        body = (await req.json()) as RagRequestBody;
+      } catch {
+        return new Response("Invalid JSON", { status: 400 });
+      }
+
+      if (!body.question || typeof body.question !== "string") {
+        return new Response('Missing required field "question"', { status: 400 });
+      }
+
+      const { question, ...rest } = body;
+
+      // Merge defaults → request body fields win
+      const answerInput: AnswerInput = {
+        ...(opts.defaults ?? {}),
+        ...(rest as Partial<AnswerInput>),
+        question,
+        kb: opts.kb,
+        chat: opts.chat,
+        embedder: opts.embedder,
+      };
+
+      const stream = new ReadableStream({
+        async start(controller) {
+          const enc = new TextEncoder();
+
+          function send(token: string) {
+            controller.enqueue(enc.encode(`data: ${token}\n\n`));
+          }
+
+          try {
+            for await (const token of answerWithRagStream(answerInput)) {
+              send(token);
+            }
+            send("[DONE]");
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            send(`[ERROR] ${msg}`);
+          } finally {
+            controller.close();
+          }
+        },
+      });
+
+      return new Response(stream, {
+        headers: {
+          "content-type": "text/event-stream",
+          "cache-control": "no-cache",
+          connection: "keep-alive",
+          ...corsHeaders(),
+        },
+      });
+    },
+  });
+}

package/src/stores/memory-store.ts

@@ -0,0 +1,249 @@
+import type { IKbStore, KbSearchHit } from "../types.ts";
+import { reciprocalRankFusion } from "../utils.ts";
+
+interface StoredDocument {
+  id: number;
+  source: string;
+  title: string;
+  contentHash: string;
+  topic: string | null;
+}
+
+interface StoredChunk {
+  chunkId: number;
+  documentId: number;
+  chunkIndex: number;
+  text: string;
+  tokenCount: number;
+  embedding: number[];
+}
+
+/**
+ * Zero-dependency in-memory `IKbStore` implementation.
+ *
+ * Suitable for:
+ * - Unit / integration tests (no database required)
+ * - Quick prototyping and examples
+ * - Demos and tutorials
+ *
+ * Limitations vs a real pgvector store:
+ * - O(n) linear scan for every search (no index) — fine up to ~50k chunks
+ * - BM25 is a minimal TF-IDF approximation, not a full BM25 implementation
+ * - Data is not persisted across process restarts
+ *
+ * @example
+ * ```ts
+ * import { InMemoryKbStore, ingestText, OllamaEmbeddingClient } from "@chatman-media/kb";
+ *
+ * const kb = new InMemoryKbStore();
+ * const embedder = new OllamaEmbeddingClient({ host: "http://localhost:11434", model: "nomic-embed-text", dim: 768 });
+ *
+ * await ingestText({ title: "FAQ", body: "..." }, { kb, embedder });
+ * const hits = await kb.search(await embedder.embed(["query"])[0], 5);
+ * ```
+ */
+export class InMemoryKbStore implements IKbStore {
+  private docs: StoredDocument[] = [];
+  private chunks: StoredChunk[] = [];
+  private nextDocId = 1;
+  private nextChunkId = 1;
+
+  // ── Search ────────────────────────────────────────────────────────────────
+
+  async search(embedding: number[], k: number, topic?: string | null): Promise<KbSearchHit[]> {
+    const pool = topic
+      ? this.chunks.filter((c) => this.topicOf(c.documentId) === topic)
+      : this.chunks;
+    return pool
+      .map((c) => ({ chunk: c, distance: cosineDistance(embedding, c.embedding) }))
+      .sort((a, b) => a.distance - b.distance)
+      .slice(0, k)
+      .map(({ chunk, distance }) => this.toHit(chunk, distance));
+  }
+
+  async hybridSearch(input: {
+    embedding: number[];
+    query: string;
+    k?: number;
+    topic?: string | null;
+  }): Promise<KbSearchHit[]> {
+    const k = input.k ?? 5;
+    const pool = input.topic
+      ? this.chunks.filter((c) => this.topicOf(c.documentId) === input.topic)
+      : this.chunks;
+
+    const vec = pool
+      .map((c) => ({ chunk: c, distance: cosineDistance(input.embedding, c.embedding) }))
+      .sort((a, b) => a.distance - b.distance)
+      .slice(0, k * 2)
+      .map(({ chunk, distance }) => this.toHit(chunk, distance));
+
+    const bm25 = simpleBm25(input.query, pool)
+      .slice(0, k * 2)
+      .map(({ chunk, score }) => this.toHit(chunk, -score));
+
+    if (bm25.length === 0) return vec.slice(0, k);
+    return reciprocalRankFusion(vec, bm25, k);
+  }
+
+  async prioritySearch(input: {
+    embedding: number[];
+    query: string;
+    k?: number;
+    vectorOnly?: boolean;
+  }): Promise<KbSearchHit[]> {
+    const k = input.k ?? 5;
+    const booksHits = input.vectorOnly
+      ? await this.search(input.embedding, k, "books")
+      : await this.hybridSearch({ ...input, k, topic: "books" });
+    if (booksHits.length > 0) return booksHits;
+    return input.vectorOnly ? this.search(input.embedding, k) : this.hybridSearch({ ...input, k });
+  }
+
+  // ── Ingest ────────────────────────────────────────────────────────────────
+
+  async getDocumentBySource(source: string): Promise<{ id: number; content_hash: string } | null> {
+    const doc = this.docs.find((d) => d.source === source);
+    return doc ? { id: doc.id, content_hash: doc.contentHash } : null;
+  }
+
+  async countChunksForDocument(documentId: number): Promise<number> {
+    return this.chunks.filter((c) => c.documentId === documentId).length;
+  }
+
+  async deleteDocument(id: number): Promise<boolean> {
+    const before = this.docs.length;
+    this.docs = this.docs.filter((d) => d.id !== id);
+    this.chunks = this.chunks.filter((c) => c.documentId !== id);
+    return this.docs.length < before;
+  }
+
+  async upsertDocument(input: {
+    source: string;
+    title: string;
+    contentHash: string;
+    topic?: string | null;
+  }): Promise<{ id: number }> {
+    const existing = this.docs.find((d) => d.source === input.source);
+    if (existing) {
+      existing.title = input.title;
+      existing.contentHash = input.contentHash;
+      existing.topic = input.topic ?? null;
+      return { id: existing.id };
+    }
+    const id = this.nextDocId++;
+    this.docs.push({
+      id,
+      source: input.source,
+      title: input.title,
+      contentHash: input.contentHash,
+      topic: input.topic ?? null,
+    });
+    return { id };
+  }
+
+  async insertChunkWithEmbedding(input: {
+    documentId: number;
+    chunkIndex: number;
+    text: string;
+    tokenCount: number;
+    embedding: number[];
+  }): Promise<void> {
+    this.chunks.push({
+      chunkId: this.nextChunkId++,
+      documentId: input.documentId,
+      chunkIndex: input.chunkIndex,
+      text: input.text,
+      tokenCount: input.tokenCount,
+      embedding: input.embedding,
+    });
+  }
+
+  /** Total number of indexed chunks. Useful in tests. */
+  get chunkCount(): number {
+    return this.chunks.length;
+  }
+
+  /** Total number of indexed documents. Useful in tests. */
+  get documentCount(): number {
+    return this.docs.length;
+  }
+
+  // ── Private ───────────────────────────────────────────────────────────────
+
+  private topicOf(documentId: number): string | null {
+    return this.docs.find((d) => d.id === documentId)?.topic ?? null;
+  }
+
+  private toHit(chunk: StoredChunk, distance: number): KbSearchHit {
+    const doc = this.docs.find((d) => d.id === chunk.documentId);
+    return {
+      chunk_id: chunk.chunkId,
+      distance,
+      text: chunk.text,
+      document_id: chunk.documentId,
+      source: doc?.source ?? "",
+      title: doc?.title ?? "",
+    };
+  }
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function cosineDistance(a: number[], b: number[]): number {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += (a[i] ?? 0) * (b[i] ?? 0);
+    normA += (a[i] ?? 0) ** 2;
+    normB += (b[i] ?? 0) ** 2;
+  }
+  if (normA === 0 || normB === 0) return 1;
+  return 1 - dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/** Minimal TF-IDF approximation — good enough for in-memory BM25 ranking. */
+function simpleBm25(
+  query: string,
+  chunks: StoredChunk[],
+): Array<{ chunk: StoredChunk; score: number }> {
+  const tokens = tokenize(query);
+  if (tokens.length === 0) return [];
+
+  const df = new Map<string, number>();
+  const N = chunks.length;
+  for (const chunk of chunks) {
+    const terms = new Set(tokenize(chunk.text));
+    for (const t of terms) df.set(t, (df.get(t) ?? 0) + 1);
+  }
+
+  return chunks
+    .map((chunk) => {
+      const words = tokenize(chunk.text);
+      const tf = new Map<string, number>();
+      for (const w of words) tf.set(w, (tf.get(w) ?? 0) + 1);
+      const dl = words.length;
+      const avgdl = 50;
+      const k1 = 1.5;
+      const b = 0.75;
+
+      let score = 0;
+      for (const t of tokens) {
+        const idf = Math.log((N - (df.get(t) ?? 0) + 0.5) / ((df.get(t) ?? 0) + 0.5) + 1);
+        const tfVal = tf.get(t) ?? 0;
+        score += idf * ((tfVal * (k1 + 1)) / (tfVal + k1 * (1 - b + b * (dl / avgdl))));
+      }
+      return { chunk, score };
+    })
+    .filter((r) => r.score > 0)
+    .sort((a, b) => b.score - a.score);
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]/gu, " ")
+    .split(/\s+/)
+    .filter((t) => t.length >= 2);
+}

package/src/structured-output.ts

@@ -0,0 +1,47 @@
+import { z } from "zod";
+
+/**
+ * Injects a JSON output instruction into the system prompt when native
+ * structured output is not available (Ollama, OpenRouter, etc.).
+ */
+export function injectJsonInstruction(
+  systemPrompt: string,
+  jsonSchema: Record<string, unknown>,
+): string {
+  const schemaStr = JSON.stringify(jsonSchema, null, 2);
+  return `${systemPrompt}\n\nRespond with a single valid JSON object matching this schema. Do not include markdown, code fences, or any text outside the JSON.\nSchema:\n${schemaStr}`;
+}
+
+/**
+ * Parses and validates an LLM response against a Zod schema.
+ * Strips markdown code fences if present before parsing.
+ * Returns `{ success: true, data }` or `{ success: false, error }`.
+ */
+export function parseStructuredOutput<T extends z.ZodTypeAny>(
+  raw: string,
+  schema: T,
+): { success: true; data: z.infer<T> } | { success: false; error: string } {
+  const cleaned = raw
+    .trim()
+    .replace(/^```(?:json)?\s*/i, "")
+    .replace(/\s*```$/, "")
+    .trim();
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    return { success: false, error: `JSON parse failed: ${cleaned.slice(0, 200)}` };
+  }
+
+  const result = schema.safeParse(parsed);
+  if (!result.success) {
+    return { success: false, error: result.error.message };
+  }
+  return { success: true, data: result.data as z.infer<T> };
+}
+
+/** Converts a Zod schema to a JSON Schema object for OpenAI's response_format. */
+export function zodToJsonSchema(schema: z.ZodTypeAny): Record<string, unknown> {
+  return z.toJSONSchema(schema) as Record<string, unknown>;
+}