@chatman-media/kb 1.3.0

package/src/tool-loop.ts

@@ -0,0 +1,110 @@
+import type { AnswerTelemetry } from "./answer-types.ts";
+import type { ChatClient, ChatCompletionOpts, ChatMessage } from "@chatman-media/llm-router";
+import type { AnyRagTool } from "./tools.ts";
+import { toolToOpenAIFunction } from "./tools.ts";
+
+/** Default maximum number of agentic tool-calling cycles. */
+export const DEFAULT_MAX_TOOL_CYCLES = 4;
+
+/** A single tool execution recorded during an agentic tool loop. */
+export interface ToolCallRecord {
+  name: string;
+  args: Record<string, unknown>;
+  result: unknown;
+  /** True when the tool was unknown or `execute()` threw — `result` holds `{ error }`. */
+  error?: boolean;
+  /** Zero-based index of the loop cycle this call belongs to. */
+  cycle: number;
+}
+
+export interface ToolLoopResult {
+  /** Final assistant text when the model stopped calling tools, else null. */
+  content: string | null;
+  /** Every tool call executed across all cycles, in order. */
+  toolCalls: ToolCallRecord[];
+  /** True when the loop stopped because `maxCycles` was hit while the model still wanted tools. */
+  exhausted: boolean;
+}
+
+/**
+ * Runs an agentic tool-calling loop. Mutates `messages` in place, appending an
+ * assistant message (carrying all tool calls) and one `tool` message per call
+ * for every cycle. Returns the final model text (when produced) and the full
+ * tool-call trace.
+ *
+ * Unknown tools and thrown `execute()` errors are fed back to the model as the
+ * tool result so it can recover — the loop never throws on tool failure.
+ *
+ * The caller must guarantee `chat.completeWithTools` exists and `tools` is non-empty.
+ */
+export async function runToolLoop(opts: {
+  chat: ChatClient;
+  messages: ChatMessage[];
+  tools: AnyRagTool[];
+  llmOpts: ChatCompletionOpts;
+  maxCycles: number;
+}): Promise<ToolLoopResult> {
+  const { chat, messages, tools, llmOpts, maxCycles } = opts;
+  const completeWithTools = chat.completeWithTools;
+  if (!completeWithTools) throw new Error("runToolLoop: chat.completeWithTools is required");
+  const toolDefs = tools.map(toolToOpenAIFunction);
+  const toolCalls: ToolCallRecord[] = [];
+
+  for (let cycle = 0; cycle < maxCycles; cycle++) {
+    const res = await completeWithTools.call(chat, messages, toolDefs, llmOpts);
+
+    if (res.toolCalls.length === 0) {
+      return { content: res.content, toolCalls, exhausted: false };
+    }
+
+    messages.push({
+      role: "assistant",
+      content: null,
+      tool_calls: res.toolCalls.map((tc) => ({
+        id: tc.id,
+        type: "function",
+        function: { name: tc.name, arguments: JSON.stringify(tc.args) },
+      })),
+    });
+
+    const settled = await Promise.allSettled(
+      res.toolCalls.map((tc) => {
+        const tool = tools.find((t) => t.name === tc.name);
+        if (!tool) return Promise.reject(new Error(`unknown tool: ${tc.name}`));
+        return tool.execute(tc.args);
+      }),
+    );
+
+    for (let i = 0; i < res.toolCalls.length; i++) {
+      const tc = res.toolCalls[i] as (typeof res.toolCalls)[number];
+      const outcome = settled[i] as PromiseSettledResult<unknown>;
+      let payload: unknown;
+      let isError = false;
+      if (outcome.status === "fulfilled") {
+        payload = outcome.value;
+      } else {
+        isError = true;
+        const message =
+          outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason);
+        payload = { error: message };
+        console.warn(`[tool-loop] tool "${tc.name}" failed: ${message}`);
+      }
+      toolCalls.push({ name: tc.name, args: tc.args, result: payload, error: isError, cycle });
+      messages.push({ role: "tool", content: JSON.stringify(payload), tool_call_id: tc.id });
+    }
+  }
+
+  return { content: null, toolCalls, exhausted: true };
+}
+
+/** Builds the tool-related telemetry fields from a tool-call trace. */
+export function buildToolTelemetry(
+  records: ToolCallRecord[],
+): Pick<AnswerTelemetry, "toolCall" | "toolCalls"> {
+  const first = records[0];
+  if (!first) return {};
+  return {
+    toolCall: { name: first.name, result: first.result },
+    toolCalls: records,
+  };
+}

package/src/tools.ts

@@ -0,0 +1,79 @@
+import { z } from "zod";
+
+/**
+ * A tool the LLM can call during `answerWithRag`.
+ *
+ * Pass one or more tools via `AnswerInput.tools`. When the model decides to
+ * use a tool, the library executes `execute()` automatically and feeds the
+ * result back to the model for a final answer (single-cycle — one tool call
+ * per request).
+ *
+ * @example
+ * ```ts
+ * import { answerWithRag, type RagTool } from "@chatman-media/kb";
+ * import { z } from "zod";
+ *
+ * const slotsTool: RagTool = {
+ *   name: "getAvailableSlots",
+ *   description: "Returns available booking slots for a given date (YYYY-MM-DD).",
+ *   parameters: z.object({ date: z.string() }),
+ *   execute: async ({ date }) => fetchSlotsFromCRM(date),
+ * };
+ *
+ * const result = await answerWithRag({ question, kb, chat, embedder, tools: [slotsTool] });
+ * // result.telemetry.toolCall — name + result if a tool was invoked
+ * ```
+ */
+export interface RagTool<TParams extends z.ZodTypeAny = z.ZodTypeAny> {
+  name: string;
+  description: string;
+  /** Zod schema for the tool's input parameters. Used to build the JSON Schema sent to the model. */
+  parameters: TParams;
+  execute: (args: z.infer<TParams>) => Promise<unknown>;
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: intentional open type for mixed tool arrays
+export type AnyRagTool = RagTool<any>;
+
+/** Converts a `RagTool` to the OpenAI function-calling format. */
+export function toolToOpenAIFunction(tool: AnyRagTool): OpenAIToolDefinition {
+  return {
+    type: "function",
+    function: {
+      name: tool.name,
+      description: tool.description,
+      parameters: z.toJSONSchema(tool.parameters) as Record<string, unknown>,
+    },
+  };
+}
+
+// ── Internal types used by ChatClient ────────────────────────────────────────
+
+export interface OpenAIToolDefinition {
+  type: "function";
+  function: {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  };
+}
+
+export interface ToolCallRequest {
+  id: string;
+  type: "function";
+  function: { name: string; arguments: string };
+}
+
+export interface ToolCallResult {
+  /** Tool call id returned by the model. */
+  id: string;
+  name: string;
+  args: Record<string, unknown>;
+}
+
+export interface CompleteWithToolsResult {
+  /** Model text when no tool was called. */
+  content: string | null;
+  /** Parsed tool calls when the model chose to call a tool. */
+  toolCalls: ToolCallResult[];
+}

package/src/topic-classifier.ts

@@ -0,0 +1,112 @@
+/**
+ * Deterministic topic classifier for inbound questions in the recruitment
+ * domain. Returns a topic slug (matching the values you tag KB docs with at
+ * ingest time) or `null` when the question doesn't fall cleanly into one
+ * bucket. No LLM call — pure regex over Russian + English with Unicode-
+ * property word boundaries (JS `\b` is ASCII-only, fails silently on
+ * Cyrillic; same trick used in stage-router.ts and rewrite-query.ts).
+ *
+ * Designed to be CONSERVATIVE: when in doubt, return null. The webhook
+ * falls back to global retrieval on null, so a missed classification just
+ * loses precision — not recall. Matching multiple topics also returns null
+ * (mixed-intent question shouldn't be force-routed).
+ *
+ * Add new topics by appending to TOPIC_PATTERNS. The slug must match the
+ * `topic` value used by your ingest pipeline (typically a directory name
+ * under `kb/curated/<slug>/`).
+ */
+// Cyrillic-aware leading word-boundary lookbehind. Trailing word-boundary
+// lookahead is intentionally OMITTED — patterns are STEMS that should match
+// any inflected form ("зарплат" matches "зарплата" / "зарплате" / etc).
+// False positives from inside other words are blocked by the leading
+// lookbehind alone (e.g. "девиз" doesn't match the "виз" stem because the
+// "е" before "виз" is \p{L}).
+const NW = `(?<![\\p{L}\\p{N}])`;
+
+const TOPIC_PATTERNS: Array<{ topic: string; pattern: RegExp }> = [
+  {
+    topic: "visa",
+    // Виза, visa, оформление документов на въезд.
+    pattern: new RegExp(`${NW}(виз|visa|загранпаспорт|invitation|приглашен)`, "iu"),
+  },
+  {
+    topic: "payment",
+    // Деньги: зарплата, оплата, ставка, комиссия, юани/евро/доллары.
+    // "плат" is intentionally excluded as a bare stem — it's too generic
+    // (matches "плата" / "платье" / "поплатишься"). Inflected verb forms
+    // ("платят", "платить") are caught by the explicit list.
+    pattern: new RegExp(
+      `${NW}(зарплат|оплат|плат[ияеу]|ставк|комисси|юан|доллар|евро|рубл|salary|payment|rate)`,
+      "iu",
+    ),
+  },
+  {
+    topic: "schedule",
+    // График, смены, часы, выходные.
+    pattern: new RegExp(`${NW}(график|смен|часов|выходн|расписан|schedule|shift)`, "iu"),
+  },
+  {
+    topic: "housing",
+    // Жильё, проживание, общежитие, квартира. Both Russian verb stems
+    // "жил-" (past) and "жит-" (infinitive) are needed.
+    pattern: new RegExp(`${NW}(жил|жит[ьея]|проживан|общежит|квартир|комнат|hous|accommod)`, "iu"),
+  },
+  {
+    topic: "locations",
+    // Конкретные локации в этом домене (Дубай, Стамбул, Китай, Корея).
+    pattern: new RegExp(
+      `${NW}(дуба|стамбул|кита|коре|шаохин|вэньчжоу|йиу|dubai|istanbul|china|korea)`,
+      "iu",
+    ),
+  },
+  {
+    topic: "vacancy",
+    // Прямые вопросы про офферы / "что у вас сейчас", виды клубов (KTV).
+    // NOT triggered by location words alone — those go to "locations".
+    pattern: new RegExp(
+      `${NW}(ваканс|оффер|какие\\s+(есть|у\\s+вас)|что\\s+у\\s+вас\\s+(есть|сеичас|сейчас)|чем\\s+(можете|можно)|ktv|караоке\\s+хостес|хостес)`,
+      "iu",
+    ),
+  },
+  {
+    topic: "requirements",
+    // Требования к кандидату: рост, вес, возраст, опыт, фото/портфолио.
+    pattern: new RegExp(
+      `${NW}(рост|вес\\b|возраст|сколько\\s+лет|портфолио|фотосет|какие\\s+треб|нужно\\s+ли\\s+знать|опыт\\s+(работ|в))`,
+      "iu",
+    ),
+  },
+  {
+    topic: "application",
+    // Анкета, форма подачи / форма для, заявка, подать.
+    pattern: new RegExp(`${NW}(анкет|форм[аыу]\\s+(подач|для)|заявк|подать|application)`, "iu"),
+  },
+];
+
+/**
+ * Returns a single topic slug when exactly ONE topic pattern matches, or
+ * `null` when zero or multiple match. Multi-match is treated as ambiguous —
+ * forcing one topic would silently drop docs from the other.
+ */
+export function classifyTopic(question: string): string | null {
+  if (!question?.trim()) return null;
+  const matches: string[] = [];
+  for (const { topic, pattern } of TOPIC_PATTERNS) {
+    if (pattern.test(question)) matches.push(topic);
+  }
+  if (matches.length === 1) return matches[0] ?? null;
+  return null;
+}
+
+/** Exposed for tests + admin debugging — lets callers see ALL matches. */
+export function classifyTopicAll(question: string): string[] {
+  if (!question?.trim()) return [];
+  const matches: string[] = [];
+  for (const { topic, pattern } of TOPIC_PATTERNS) {
+    if (pattern.test(question)) matches.push(topic);
+  }
+  return matches;
+}
+
+/** All defined topic slugs. Useful for ingest CLI validation and admin UI. */
+export const KNOWN_TOPICS: readonly string[] = TOPIC_PATTERNS.map((p) => p.topic);

package/src/types.ts

@@ -0,0 +1,91 @@
+/**
+ * Storage interfaces for the RAG engine.
+ *
+ * The package defines these interfaces; consumers provide implementations.
+ * The reference implementation (`PgKbStore`) ships with the lead-engine platform
+ * project and wraps the PostgreSQL + pgvector repos.
+ */
+
+export interface KbSearchHit {
+  chunk_id: number;
+  /** Cosine distance (vector search) or negated BM25 rank (FTS). Lower = closer. */
+  distance: number;
+  text: string;
+  document_id: number;
+  source: string;
+  title: string;
+}
+
+/**
+ * Minimal contract the RAG engine needs from whatever storage backend you
+ * plug in. Split into two logical groups:
+ *
+ * - **Search** — called by `answerWithRag` at query time (read-only).
+ * - **Ingest** — called by `ingestFile` / `ingestText` / `ingestDirectory`
+ *   when indexing documents (read+write).
+ */
+export interface IKbStore {
+  // ── Search ──────────────────────────────────────────────────────────────────
+
+  /**
+   * Pure vector search (cosine distance via pgvector or equivalent).
+   * Returns up to `k` hits, optionally filtered by `topic`.
+   */
+  search(embedding: number[], k: number, topic?: string | null): Promise<KbSearchHit[]>;
+
+  /**
+   * Hybrid retrieval: fuse vector + BM25 results via Reciprocal Rank Fusion.
+   * Falls back to vector-only when the FTS index returns nothing.
+   */
+  hybridSearch(input: {
+    embedding: number[];
+    query: string;
+    k?: number;
+    topic?: string | null;
+  }): Promise<KbSearchHit[]>;
+
+  /**
+   * Books-priority search: tries the "books" topic first, falls back to
+   * global search when no books-tagged chunks match. Used when `booksPriority`
+   * is set on `AnswerInput`.
+   */
+  prioritySearch(input: {
+    embedding: number[];
+    query: string;
+    k?: number;
+    vectorOnly?: boolean;
+  }): Promise<KbSearchHit[]>;
+
+  // ── Ingest ──────────────────────────────────────────────────────────────────
+
+  /** Look up an existing document by its source URI. */
+  getDocumentBySource(source: string): Promise<{ id: number; content_hash: string } | null>;
+
+  /** Count indexed chunks for a document (used for dedup short-circuit). */
+  countChunksForDocument(documentId: number): Promise<number>;
+
+  /** Delete a document and all its chunks (called before re-indexing). */
+  deleteDocument(id: number): Promise<boolean>;
+
+  /** Upsert a document record; returns the canonical row id. */
+  upsertDocument(input: {
+    source: string;
+    title: string;
+    contentHash: string;
+    topic?: string | null;
+  }): Promise<{ id: number }>;
+
+  /** Insert one chunk with its embedding vector. */
+  insertChunkWithEmbedding(input: {
+    documentId: number;
+    chunkIndex: number;
+    text: string;
+    tokenCount: number;
+    embedding: number[];
+  }): Promise<void>;
+}
+
+/** Optional write-only interface for logging unanswered questions. */
+export interface IKbSuggestionsStore {
+  log(question: string, conversationId: string): Promise<void>;
+}

package/src/utils.ts

@@ -0,0 +1,81 @@
+import type { KbSearchHit } from "./types.ts";
+
+/**
+ * Reciprocal Rank Fusion — fuses two ranked result lists into one.
+ *
+ * Standard formula: score(d) = Σ 1/(k + rank(d))
+ * where k=60 is the smoothing constant (Cormack et al., 2009).
+ *
+ * Returns hits sorted by descending fused score. The `distance` field is
+ * remapped to `1 - score` so callers get a consistent "lower = better"
+ * interpretation regardless of whether the underlying search was vector or BM25.
+ *
+ * @param vectorHits  Results from vector (cosine) search, ranked by ascending distance.
+ * @param bm25Hits    Results from BM25 search, ranked by descending BM25 score.
+ * @param k           Number of top results to return.
+ * @param rrfK        RRF smoothing constant (default 60).
+ */
+export function reciprocalRankFusion(
+  vectorHits: KbSearchHit[],
+  bm25Hits: KbSearchHit[],
+  k: number,
+  rrfK = 60,
+): KbSearchHit[] {
+  if (bm25Hits.length === 0) return vectorHits.slice(0, k);
+  if (vectorHits.length === 0) return bm25Hits.slice(0, k);
+
+  const scores = new Map<number, { hit: KbSearchHit; score: number }>();
+
+  const addRanked = (hits: KbSearchHit[]) => {
+    hits.forEach((h, i) => {
+      const rank = i + 1;
+      const inc = 1 / (rrfK + rank);
+      const prev = scores.get(h.chunk_id);
+      if (prev) {
+        prev.score += inc;
+      } else {
+        scores.set(h.chunk_id, { hit: h, score: inc });
+      }
+    });
+  };
+
+  addRanked(vectorHits);
+  addRanked(bm25Hits);
+
+  return Array.from(scores.values())
+    .sort((a, b) => b.score - a.score)
+    .slice(0, k)
+    .map(({ hit, score }) => ({ ...hit, distance: 1 - score }));
+}
+
+/**
+ * Sanitizes a raw user query into a valid PostgreSQL `tsquery` string for
+ * Russian full-text search. Uses prefix-OR semantics (`term:* OR term:*`)
+ * so partial words and morphological variants match without a stemming dict.
+ *
+ * Strips tsquery operator characters to prevent injection:
+ * `"`, `'`, `(`, `)`, `*`, `:`, `.`, `\\`, `^`, `&`, `|`, `!`
+ * and boolean keywords `AND`, `OR`, `NOT`, `NEAR`.
+ *
+ * Returns an empty string when the query contains no usable tokens (caller
+ * should skip the FTS query entirely in that case).
+ *
+ * @example
+ * sanitizeFtsQuery("виза оформляется")
+ * // → "виза:* | оформляется:*"
+ *
+ * sanitizeFtsQuery('OR "injection"')
+ * // → "injection:*"
+ */
+const FTS_KEYWORDS = new Set(["and", "or", "not", "near"]);
+
+export function sanitizeFtsQuery(raw: string): string {
+  if (!raw) return "";
+  const stripped = raw.replace(/["'()*:.\\^&|!]/g, " ");
+  const tokens = stripped
+    .split(/\s+/)
+    .map((t) => t.trim())
+    .filter((t) => t.length >= 2 && !FTS_KEYWORDS.has(t.toLowerCase()));
+  if (tokens.length === 0) return "";
+  return tokens.map((t) => `${t}:*`).join(" | ");
+}