@chatman-media/kb 1.3.0

package/src/fact-checker.ts

@@ -0,0 +1,171 @@
+import type { ChatClient, ChatMessage } from "@chatman-media/llm-router";
+import { stripCodeFences, stripThinkBlocks } from "./sanitize.ts";
+
+/**
+ * Unified fact-checker — replaces the separate reflect.ts + vacancy-guard.ts
+ * pipeline with a single LLM call that checks both:
+ *
+ *   1. KB grounding: every concrete fact in the answer is supported by the
+ *      retrieved KB context (no hallucinations).
+ *
+ *   2. Vacancy accuracy: any salary/city/condition data matches the
+ *      authoritative АКТУАЛЬНЫЕ ВАКАНСИИ block from the DB table.
+ *      ВАКАНСИИ always win over KB chunks when they conflict — an old
+ *      chat log with a stale salary is not a valid grounding source.
+ *
+ * Cost: 1 LLM call per turn (was 2 with separate reflect + vacancy-guard).
+ *
+ * **Fail-closed by default**: on any LLM or parse error returns
+ * `{ grounded: false, vacancyOk: false, reason: "checker_error: …" }` so the
+ * caller treats the reply as ungrounded and escalates to silence/operator
+ * instead of shipping a potentially-hallucinated answer. The legacy
+ * fail-open behaviour can be restored with `RAG_FACT_CHECKER_FAIL_OPEN=1`
+ * (kept for test fixtures that drive the checker with mock failures).
+ */
+
+export interface FactCheckInput {
+  question: string;
+  answer: string;
+  /** KB chunks concatenated — same string passed to the generator. */
+  context: string;
+  chat: ChatClient;
+  /** Optional: rendered АКТУАЛЬНЫЕ ВАКАНСИИ block.
+   *  When provided, vacancy facts are checked against it as the authoritative
+   *  source. KB chunks that contradict it are treated as outdated. */
+  vacanciesBlock?: string;
+}
+
+export interface FactCheckResult {
+  /** All concrete facts in the answer are found in context (or vacanciesBlock). */
+  grounded: boolean;
+  /** All vacancy-specific facts (salary, city, conditions) match vacanciesBlock.
+   *  Always true when vacanciesBlock was not provided. */
+  vacancyOk: boolean;
+  /** Human-readable reason when grounded=false or vacancyOk=false. */
+  reason?: string;
+}
+
+// ── System prompts ──────────────────────────────────────────────────────────
+
+const SYSTEM_PROMPT_NO_VACANCIES = `Ты проверяешь ответ бота на галлюцинации.
+Тебе дают: ВОПРОС кандидата, КОНТЕКСТ (выдержки из базы знаний), ОТВЕТ бота.
+
+Задача: проверить, что КАЖДЫЙ конкретный факт из ОТВЕТА встречается в КОНТЕКСТЕ.
+
+Правила:
+- Общие фразы ("хорошие условия", "напишу подробности"), приветствия, эмоции — НЕ требуют проверки
+- Конкретные числа, страны, города, валюты, сроки, % — ДОЛЖНЫ быть в контексте
+- "уточню у руководства" / "уточним на созвоне" — ОК, не требует проверки
+- Личные факты бота (имя, возраст, город, статус) — НЕ требуют проверки в контексте
+
+Верни СТРОГО JSON одной строкой, без markdown, без \`\`\`:
+{"grounded": true, "vacancyOk": true} — если все факты подтверждены или ответ общий
+{"grounded": false, "vacancyOk": true, "reason": "<какой факт не из контекста>"} — если есть выдуманное
+
+Только JSON, ничего больше.`;
+
+const SYSTEM_PROMPT_WITH_VACANCIES = `Ты проверяешь ответ бота на два типа ошибок.
+Тебе дают: ВОПРОС кандидата, KB-КОНТЕКСТ (выдержки из базы знаний), АКТУАЛЬНЫЕ ВАКАНСИИ (официальный список из БД), ОТВЕТ бота.
+
+ВАЖНО: АКТУАЛЬНЫЕ ВАКАНСИИ имеют приоритет над KB-КОНТЕКСТОМ.
+Если KB-чанк содержит устаревшую зарплату/условие, а в АКТУАЛЬНЫХ ВАКАНСИЯХ другое — правы ВАКАНСИИ.
+
+Проверка 1 — KB grounding:
+- Все конкретные факты в ОТВЕТЕ (числа, страны, города, валюты, сроки, %) должны быть в KB-КОНТЕКСТЕ или в АКТУАЛЬНЫХ ВАКАНСИЯХ.
+- Общие фразы, приветствия, "уточним на созвоне", личные факты бота — НЕ проверяются.
+
+Проверка 2 — Vacancy accuracy (только если ОТВЕТ содержит конкретные данные о вакансиях):
+- Суммы заработка, города трудоустройства, условия жилья/перелёта, тип заведения —
+  должны совпадать с АКТУАЛЬНЫМИ ВАКАНСИЯМИ.
+- Общие фразы ("легальный контракт", "поддержка агентства") — НЕ проверяются.
+
+Верни СТРОГО JSON одной строкой, без markdown, без \`\`\`:
+{"grounded": true, "vacancyOk": true} — всё ок
+{"grounded": false, "vacancyOk": true, "reason": "..."} — выдуманный факт (не из KB и не из вакансий)
+{"grounded": true, "vacancyOk": false, "reason": "..."} — данные вакансии не совпадают с официальным списком
+{"grounded": false, "vacancyOk": false, "reason": "..."} — оба нарушения
+
+Только JSON, ничего больше.`;
+
+// ── Main function ───────────────────────────────────────────────────────────
+
+/** Reply when the checker itself cannot run (LLM error / network).
+ *  Default: treat the answer as ungrounded so it gets silenced. The
+ *  RAG_FACT_CHECKER_FAIL_OPEN=1 env override restores the legacy
+ *  "let it through" behaviour for test environments that intentionally
+ *  inject failing LLM mocks. */
+function checkerErrorResult(reason: string): FactCheckResult {
+  if (process.env.RAG_FACT_CHECKER_FAIL_OPEN === "1") {
+    return { grounded: true, vacancyOk: true };
+  }
+  return { grounded: false, vacancyOk: false, reason: `checker_error: ${reason}` };
+}
+
+export async function checkFacts(input: FactCheckInput): Promise<FactCheckResult> {
+  const OK: FactCheckResult = { grounded: true, vacancyOk: true };
+
+  const trimmed = input.answer.trim();
+  if (!trimmed) return OK;
+
+  const hasContext = input.context.trim().length > 0;
+  const hasVacancies = (input.vacanciesBlock ?? "").trim().length > 0;
+
+  // Nothing to check against — let it through (generator's own NO_CONTEXT
+  // rules are the last line of defense on truly empty turns).
+  if (!hasContext && !hasVacancies) return OK;
+
+  const systemPrompt = hasVacancies ? SYSTEM_PROMPT_WITH_VACANCIES : SYSTEM_PROMPT_NO_VACANCIES;
+
+  let userPrompt: string;
+  if (hasVacancies) {
+    userPrompt =
+      `ВОПРОС: ${input.question}\n\n` +
+      `KB-КОНТЕКСТ:\n${input.context || "(пусто)"}\n\n` +
+      `АКТУАЛЬНЫЕ ВАКАНСИИ:\n${input.vacanciesBlock}\n\n` +
+      `ОТВЕТ: ${trimmed}\n\nJSON:`;
+  } else {
+    userPrompt = `ВОПРОС: ${input.question}\n\nКОНТЕКСТ:\n${input.context}\n\nОТВЕТ: ${trimmed}\n\nJSON:`;
+  }
+
+  const messages: ChatMessage[] = [
+    { role: "system", content: systemPrompt },
+    { role: "user", content: userPrompt },
+  ];
+
+  let raw: string;
+  try {
+    raw = await input.chat.complete(messages, { temperature: 0.0 });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.error("[fact-checker] LLM call failed:", err);
+    return checkerErrorResult(`llm_call: ${msg}`);
+  }
+
+  return parseFactCheckResult(raw);
+}
+
+/** Exported for unit tests. */
+export function parseFactCheckResult(raw: string): FactCheckResult {
+  const OK: FactCheckResult = { grounded: true, vacancyOk: true };
+
+  const s = stripCodeFences(stripThinkBlocks(raw)).trim();
+  const start = s.indexOf("{");
+  const end = s.lastIndexOf("}");
+  if (start === -1 || end === -1 || end < start) return OK;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(s.slice(start, end + 1));
+  } catch {
+    return OK;
+  }
+
+  if (typeof parsed !== "object" || parsed === null) return OK;
+  const obj = parsed as Record<string, unknown>;
+
+  const grounded = typeof obj.grounded === "boolean" ? obj.grounded : true;
+  const vacancyOk = typeof obj.vacancyOk === "boolean" ? obj.vacancyOk : true;
+  const reason = typeof obj.reason === "string" ? obj.reason : undefined;
+
+  return { grounded, vacancyOk, ...(reason ? { reason } : {}) };
+}

package/src/grade-skills.ts

@@ -0,0 +1,79 @@
+/**
+ * After the bot has generated a reply, optionally ask a small LLM to
+ * inspect the (system_prompt → user → assistant) triple and report which
+ * of the configured skills (slugs) it actually used. Output goes into
+ * `messages.meta_json.telemetry.skills_used` for downstream attribution
+ * (Phase 2: ELO over outcomes; Phase 3: self-play).
+ *
+ * Cost: +1 LLM call per assistant turn. Gated behind RAG_SKILL_GRADING=true
+ * because for high-volume single-style production it's overkill — flip on
+ * during A/B research windows, off in steady state.
+ *
+ * Failure-soft: any error → returns empty array, never throws. The whole
+ * point is post-hoc analytics, not a hard dependency.
+ */
+import type { ChatClient } from "@chatman-media/llm-router";
+
+export interface GradeSkillsInput {
+  question: string;
+  reply: string;
+  /** Slugs offered to the model in the system prompt (only these are valid). */
+  availableSlugs: readonly string[];
+  chat: ChatClient;
+  /** Lightweight model id — falls back to the chat client's default if undefined. */
+  model?: string;
+}
+
+const SYSTEM = (slugs: readonly string[]) =>
+  `You are a sales-conversation auditor. Given a candidate's question and a salesperson's reply, identify which persuasion skills (from the allowed list below) the reply actually demonstrates.\n\nALLOWED SKILLS:\n${slugs.map((s) => `- ${s}`).join("\n")}\n\nReturn ONLY a JSON array of slugs from the allowed list. Empty array if none clearly apply. No commentary, no markdown, no explanation. Be conservative: only include a slug when the reply demonstrably USES the technique, not just touches its theme.\n\nCue patterns (when each skill is "demonstrably USED"):\n  social-proof-stat: explicit number/% of past results ("70% наших девочек закрывают за 2 недели") → include.\n  tactical-empathy: names candidate's emotion verbatim ("звучит, что ты переживаешь о..." / "кажется, тебе важна стабильность") → include.\n  calibrated-question: open "как / что / почему" question that can't be answered yes/no ("как ты сейчас представляешь идеальный вариант?") → include.\n  liking-genuine-compliment: a SPECIFIC sincere compliment to the candidate (energy / style / sharp question / good русский) — NOT generic "красавица". Even one short line counts ("вижу, ты задаёшь правильные вопросы — это редкость" / "по фото — энергия чувствуется"). → include.\n  accusation-audit: salesperson VOICES the candidate's fear/objection BEFORE the candidate raises it ("наверное это звучит как развод" / "понимаю, кажется, что слишком хорошо чтобы быть правдой" / "догадываюсь, ты думаешь — а вдруг кинут") → include.\n  authority-license: cites concrete legal/contractual mechanism ("договор подписывается ДО вылета" / "виза оформляется агентством на их стороне" / "официальный контракт с работодателем" / "никаких выплат в счёт работы") → include.\n  scarcity-spots-left: explicit limited-slots phrasing ("3-5 мест на поток" / "разбирают за неделю" / "ближайший вылет почти набран") → include.\n  reciprocity-free-info: shares useful info WITHOUT asking for commitment in the same breath ("кстати, в Сеуле сейчас сезон — поэтому ставки выше") → include.\n  unity-belonging: uses "мы / наши девочки / наша команда" framing instead of impersonal "у нас" ("наши девочки в Шанхае пишут что..." / "мы тебя сопровождаем до прилёта") → include.\n  commitment-microyes: stacks 2+ short confirmations before a bigger ask ("21? ок. паспорт есть? ок. готова на 3 месяца?") → include.\n  mirroring: repeats candidate's last 1-3 words as a question ("не уверена?" / "слишком далеко?") → include.\n  Reply just answers a factual question with no persuasion technique → return [].`;
+
+export async function gradeSkills(input: GradeSkillsInput): Promise<string[]> {
+  if (input.availableSlugs.length === 0) return [];
+  try {
+    const raw = await input.chat.complete(
+      [
+        { role: "system", content: SYSTEM(input.availableSlugs) },
+        {
+          role: "user",
+          content: `Candidate: ${input.question}\n\nSalesperson reply: ${input.reply}\n\nReturn JSON array of skill slugs used (subset of allowed list).`,
+        },
+      ],
+      {
+        temperature: 0,
+        ...(input.model ? { model: input.model } : {}),
+        numPredict: 80,
+      },
+    );
+    return parseSlugList(raw, input.availableSlugs);
+  } catch (err) {
+    console.warn("[grade-skills] LLM call failed:", err);
+    return [];
+  }
+}
+
+/** Tolerant parser — accepts a bare JSON array, code-fenced JSON, or
+ *  comma-separated text. Filters to allowed slugs. Exported for tests. */
+export function parseSlugList(raw: string, allowed: readonly string[]): string[] {
+  if (!raw) return [];
+  const allowSet = new Set(allowed);
+  // Try JSON first.
+  const stripped = raw
+    .replace(/^```(?:json)?\s*/i, "")
+    .replace(/\s*```\s*$/i, "")
+    .trim();
+  try {
+    const parsed = JSON.parse(stripped);
+    if (Array.isArray(parsed)) {
+      return parsed
+        .filter((s): s is string => typeof s === "string")
+        .map((s) => s.trim())
+        .filter((s) => allowSet.has(s));
+    }
+  } catch {
+    /* fall through to plain-text split */
+  }
+  return stripped
+    .split(/[,\n]+/)
+    .map((s) => s.trim().replace(/^["'-]+|["'-]+$/g, ""))
+    .filter((s) => allowSet.has(s));
+}

package/src/index.ts

@@ -0,0 +1,191 @@
+/**
+ * @chatman-media/kb — production-grade RAG engine for conversational bots.
+ *
+ * Quick start:
+ *
+ * ```ts
+ * import { answerWithRag, OpenAIChatClient, OpenAIEmbeddingClient } from "@chatman-media/kb";
+ *
+ * const chat = new OpenAIChatClient({ apiKey, baseUrl, model: "gpt-4o-mini" });
+ * const embedder = new OpenAIEmbeddingClient({ apiKey, baseUrl, model: "text-embedding-3-small", dim: 1536 });
+ *
+ * const result = await answerWithRag({ question, kb: myKbStore, chat, embedder });
+ * console.log(result.text);
+ * ```
+ *
+ * See README.md for full usage and `IKbStore` implementation guide.
+ */
+
+// ── A/B style router ──────────────────────────────────────────────────────────
+export type { ABRouterOptions, ABVariant } from "./ab-router.ts";
+export { ABRouter } from "./ab-router.ts";
+// ── Core answer pipeline ─────────────────────────────────────────────────────
+export {
+  answerWithRag,
+  answerWithRagStream,
+  generateSoftFallback,
+  retrieveHits,
+  type RetrievalResult,
+} from "./answer.ts";
+export type { AnswerInput, AnswerResult, AnswerTelemetry, Persona } from "./answer-types.ts";
+export { NO_CONTEXT_MARKER } from "./answer-types.ts";
+// ── LLM clients (re-exports из @chatman-media/llm-router для backwards-compat) ─
+// Новый код должен импортировать chat/embed/providers напрямую из llm-router;
+// эти re-exports будут удалены в будущем PR после полной миграции consumers.
+export type {
+  ChatClient,
+  ChatCompletionOpts,
+  ChatMessage,
+  ChatRole,
+} from "@chatman-media/llm-router";
+export { ChatApiError, OpenAIChatClient } from "@chatman-media/llm-router";
+export type { Chunk, ChunkOptions, SectionChunk } from "./chunk.ts";
+// ── Chunking ──────────────────────────────────────────────────────────────────
+export { chunkBySections, chunkText, estimateTokens } from "./chunk.ts";
+// ── Conversation store ────────────────────────────────────────────────────────
+export type { IConversationStore } from "./conversation-store.ts";
+export { InMemoryConversationStore } from "./conversation-store.ts";
+export type { EmbeddingClient } from "@chatman-media/llm-router";
+export {
+  EmbeddingApiError,
+  NullEmbeddingClient,
+  OpenAIEmbeddingClient,
+} from "@chatman-media/llm-router";
+// ── Retrieval evaluation ──────────────────────────────────────────────────────
+export type { EvalQuery, EvalResult, QueryMetrics } from "./eval.ts";
+export { evalRetrieval } from "./eval.ts";
+export type { ExtractFactsInput } from "./extract-user-facts.ts";
+// ── Memory & conversation management ─────────────────────────────────────────
+export { extractUserFacts, parseFactsFromLlmOutput } from "./extract-user-facts.ts";
+export type { FactCheckInput, FactCheckResult } from "./fact-checker.ts";
+// ── Hallucination guard ───────────────────────────────────────────────────────
+export { checkFacts, parseFactCheckResult } from "./fact-checker.ts";
+export type { GradeSkillsInput } from "./grade-skills.ts";
+// ── Skill grading (post-hoc analytics) ───────────────────────────────────────
+export { gradeSkills } from "./grade-skills.ts";
+export type { IngestDeps, IngestDirectorySummary, IngestFileResult } from "./ingest.ts";
+// ── Ingest pipeline ───────────────────────────────────────────────────────────
+export {
+  deriveTopicFromPath,
+  ingestDirectory,
+  ingestFile,
+  ingestText,
+  stripNonContent,
+} from "./ingest.ts";
+// ── PDF parsing ───────────────────────────────────────────────────────────────
+export { parsePdf, parsePdfBuffer } from "./parse-pdf.ts";
+// ── Persona shortcuts ─────────────────────────────────────────────────────────
+export {
+  botPresenceReply,
+  isBotPresenceQuestion,
+  isPersonalFactQuestion,
+  isPersonaSmalltalkQuestion,
+  personaFactReply,
+  personaSmalltalkReply,
+} from "./persona-shortcuts.ts";
+// ── Sales-style prompt engine ────────────────────────────────────────────────
+export { composeSystemPrompt } from "./prompt.ts";
+// ── Provider implementations (re-exports из llm-router) ──────────────────────
+export type {
+  OllamaChatOptions,
+  OllamaEmbeddingOptions,
+  OpenRouterChatOptions,
+} from "@chatman-media/llm-router";
+export {
+  OllamaChatClient,
+  OllamaEmbeddingClient,
+  OpenRouterChatClient,
+} from "@chatman-media/llm-router";
+export type { ReflectInput, ReflectResult } from "./reflect.ts";
+export { parseReflection, verifyAnswer } from "./reflect.ts";
+// ── Reranker ──────────────────────────────────────────────────────────────────
+export type { CohereRerankerOptions, JinaRerankerOptions, Reranker } from "./reranker.ts";
+export { CohereReranker, JinaReranker } from "./reranker.ts";
+// ── Retry / resilience ────────────────────────────────────────────────────────
+export type { RetryOptions } from "./retry.ts";
+export { withRetryChatClient, withRetryEmbeddingClient } from "./retry.ts";
+export type { RewriteQueryInput } from "./rewrite-query.ts";
+// ── Retrieval enhancements ────────────────────────────────────────────────────
+export { questionNeedsRewrite, rewriteQuery, sanitizeRewritten } from "./rewrite-query.ts";
+// ── Output sanitization ───────────────────────────────────────────────────────
+export { sanitizeLlmOutput } from "./sanitize.ts";
+// ── Semantic cache ────────────────────────────────────────────────────────────
+export type { SemanticCacheOptions } from "./semantic-cache.ts";
+export { SemanticCache } from "./semantic-cache.ts";
+// ── SSE server ────────────────────────────────────────────────────────────────
+export type { RagRequestBody, RagServerOptions } from "./server.ts";
+export { createRagServer } from "./server.ts";
+export { InMemoryKbStore } from "./stores/memory-store.ts";
+// ── Structured output ─────────────────────────────────────────────────────────
+export { parseStructuredOutput, zodToJsonSchema } from "./structured-output.ts";
+export type {
+  ComposeOptions,
+  DirectorHookForPrompt,
+  FunnelStage,
+  Hook,
+  HookKind,
+  SalesFramework,
+  SkillForPrompt,
+  StageConfig,
+  Style,
+  StylePersona,
+} from "./styles.ts";
+export {
+  FUNNEL_STAGES,
+  HookSchema,
+  PersonaSchema,
+  SALES_FRAMEWORKS,
+  StyleSchema,
+} from "./styles.ts";
+export type { SummarizeInput } from "./summarize-conversation.ts";
+export { cleanSummary, summarizeConversation } from "./summarize-conversation.ts";
+// ── Legacy / simple prompt builder ───────────────────────────────────────────
+export {
+  buildSystemPrompt,
+  DEFAULT_PERSONA,
+  legacyRagSamplingTemperature,
+  renderSummaryBlock,
+  renderUserFactsBlock,
+} from "./system-prompt.ts";
+export type { TextStyleRule } from "./text-style-rules.ts";
+export {
+  applyStyleRules,
+  capitalizeFirstLetter,
+  DEFAULT_STYLE_RULES,
+  replaceEllipsis,
+  replaceEmDash,
+  stripAILeadIns,
+  stripMarkdownBold,
+} from "./text-style-rules.ts";
+// ── Tool calling ──────────────────────────────────────────────────────────────
+export type { AnyRagTool, CompleteWithToolsResult, RagTool } from "./tools.ts";
+// ── Built-in tools (ready-made RagTool factories) ────────────────────────────
+export { makeBookingLinkTool } from "./built-in-tools/calendly.ts";
+// ── Agentic tool-calling loop (multi-cycle, used internally by answerWithRag) ─
+export {
+  buildToolTelemetry,
+  DEFAULT_MAX_TOOL_CYCLES,
+  runToolLoop,
+  type ToolCallRecord,
+  type ToolLoopResult,
+} from "./tool-loop.ts";
+// ── Topic routing ─────────────────────────────────────────────────────────────
+export { classifyTopic, classifyTopicAll, KNOWN_TOPICS } from "./topic-classifier.ts";
+// ── Storage interfaces & implementations ─────────────────────────────────────
+export type { IKbStore, IKbSuggestionsStore, KbSearchHit } from "./types.ts";
+// ── Utilities ─────────────────────────────────────────────────────────────────
+export { reciprocalRankFusion, sanitizeFtsQuery } from "./utils.ts";
+// ── Vision / photo classification ────────────────────────────────────────────
+export type {
+  ClassifyPhotoOptions,
+  PassportIdentity,
+  PhotoClass,
+  VisionProvider,
+} from "./vision.ts";
+export {
+  classifyPhoto,
+  extractPassportIdentity,
+  PHOTO_CLASSES,
+  parsePassportJson,
+  parsePhotoClass,
+} from "./vision.ts";

package/src/ingest.ts

@@ -0,0 +1,193 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync } from "node:fs";
+import { basename, dirname, extname, join, relative, resolve } from "node:path";
+import { type ChunkOptions, chunkText } from "./chunk.ts";
+import type { EmbeddingClient } from "@chatman-media/llm-router";
+import { parsePdf } from "./parse-pdf.ts";
+import type { IKbStore } from "./types.ts";
+
+const SUPPORTED_EXTS = new Set([".md", ".txt", ".pdf"]);
+
+export interface IngestDeps {
+  kb: IKbStore;
+  embedder: EmbeddingClient;
+  chunk?: Partial<ChunkOptions>;
+  topic?: string | null;
+  /** Overrides the default `file://<abs>` document source. Callers that
+   *  ingest from a throwaway temp path (e.g. an admin upload route) pass a
+   *  stable, content-addressed source here so re-uploads dedupe. */
+  source?: string;
+  /** Overrides the default `basename(abs)` document title — useful when the
+   *  on-disk filename is a temp name rather than something operator-facing. */
+  title?: string;
+}
+
+export interface IngestFileResult {
+  source: string;
+  documentId: number;
+  chunks: number;
+  created: boolean;
+}
+
+export async function ingestFile(path: string, deps: IngestDeps): Promise<IngestFileResult> {
+  const abs = resolve(path);
+  const ext = extname(abs).toLowerCase();
+  const raw = ext === ".pdf" ? await parsePdf(abs) : readFileSync(abs, "utf8");
+  const hash = createHash("sha256").update(raw).digest("hex");
+  const source = deps.source ?? `file://${abs}`;
+  const title = deps.title ?? basename(abs);
+
+  const existing = await deps.kb.getDocumentBySource(source);
+
+  if (existing && existing.content_hash === hash) {
+    const chunks = await deps.kb.countChunksForDocument(existing.id);
+    if (chunks > 0) {
+      return { source, documentId: existing.id, chunks, created: false };
+    }
+  }
+
+  if (existing) {
+    await deps.kb.deleteDocument(existing.id);
+  }
+
+  const doc = await deps.kb.upsertDocument({
+    source,
+    title,
+    contentHash: hash,
+    ...(deps.topic !== undefined ? { topic: deps.topic } : {}),
+  });
+
+  const preparedText = ext === ".pdf" ? raw : stripNonContent(raw);
+  const chunks = chunkText(preparedText, deps.chunk);
+  if (chunks.length === 0) {
+    return { source, documentId: doc.id, chunks: 0, created: true };
+  }
+
+  const vectors = await deps.embedder.embed(chunks.map((c) => c.text));
+  for (const [i, chunk] of chunks.entries()) {
+    const vec = vectors[i];
+    if (!vec) throw new Error(`embedder returned no vector for chunk ${i}`);
+    await deps.kb.insertChunkWithEmbedding({
+      documentId: doc.id,
+      chunkIndex: chunk.index,
+      text: chunk.text,
+      tokenCount: chunk.tokenCount,
+      embedding: vec,
+    });
+  }
+  return { source, documentId: doc.id, chunks: chunks.length, created: true };
+}
+
+/**
+ * Ingest a raw text document directly (no filesystem). Used by admin-UI
+ * paste path. Source is synthesised as `inline:<sha256[:12]>` so subsequent
+ * uploads of identical content dedupe via the `source` UNIQUE lookup.
+ */
+export async function ingestText(
+  input: { title: string; body: string },
+  deps: IngestDeps,
+): Promise<IngestFileResult> {
+  const raw = input.body;
+  const hash = createHash("sha256").update(raw).digest("hex");
+  const source = `inline:${hash.slice(0, 12)}`;
+  const title = input.title.trim() || "untitled";
+
+  const existing = await deps.kb.getDocumentBySource(source);
+
+  if (existing && existing.content_hash === hash) {
+    const chunks = await deps.kb.countChunksForDocument(existing.id);
+    if (chunks > 0) {
+      return { source, documentId: existing.id, chunks, created: false };
+    }
+  }
+
+  if (existing) {
+    await deps.kb.deleteDocument(existing.id);
+  }
+
+  const doc = await deps.kb.upsertDocument({
+    source,
+    title,
+    contentHash: hash,
+    ...(deps.topic !== undefined ? { topic: deps.topic } : {}),
+  });
+
+  const chunks = chunkText(stripNonContent(raw), deps.chunk);
+  if (chunks.length === 0) {
+    return { source, documentId: doc.id, chunks: 0, created: true };
+  }
+
+  const vectors = await deps.embedder.embed(chunks.map((c) => c.text));
+  for (const [i, chunk] of chunks.entries()) {
+    const vec = vectors[i];
+    if (!vec) throw new Error(`embedder returned no vector for chunk ${i}`);
+    await deps.kb.insertChunkWithEmbedding({
+      documentId: doc.id,
+      chunkIndex: chunk.index,
+      text: chunk.text,
+      tokenCount: chunk.tokenCount,
+      embedding: vec,
+    });
+  }
+  return { source, documentId: doc.id, chunks: chunks.length, created: true };
+}
+
+export interface IngestDirectorySummary {
+  documents: number;
+  chunks: number;
+  skipped: number;
+}
+
+export async function ingestDirectory(
+  dir: string,
+  deps: IngestDeps,
+): Promise<IngestDirectorySummary> {
+  const summary: IngestDirectorySummary = { documents: 0, chunks: 0, skipped: 0 };
+  const root = resolve(dir);
+  for (const file of walk(root)) {
+    if (!SUPPORTED_EXTS.has(extname(file).toLowerCase())) {
+      summary.skipped++;
+      continue;
+    }
+    const fileDeps: IngestDeps =
+      deps.topic !== undefined ? deps : { ...deps, topic: deriveTopicFromPath(file, root) };
+    const r = await ingestFile(file, fileDeps);
+    summary.documents++;
+    summary.chunks += r.chunks;
+  }
+  return summary;
+}
+
+/**
+ * Returns the immediate sub-directory name of `file` relative to `root`,
+ * or null when the file sits directly under `root`. Exported for unit tests.
+ */
+export function deriveTopicFromPath(file: string, root: string): string | null {
+  const rel = relative(root, dirname(file));
+  if (!rel || rel === "." || rel.startsWith("..")) return null;
+  const first = rel.split(/[/\\]/)[0];
+  return first || null;
+}
+
+function* walk(dir: string): Generator<string> {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      yield* walk(full);
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}
+
+/**
+ * Removes YAML frontmatter and HTML comments from Markdown — noise that
+ * pollutes embeddings. The visible body is preserved as-is.
+ */
+export function stripNonContent(raw: string): string {
+  let s = raw;
+  s = s.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, "");
+  s = s.replace(/<!--[\s\S]*?-->/g, "");
+  s = s.replace(/\n{3,}/g, "\n\n").trim();
+  return `${s}\n`;
+}