@tryformation/querylight-cli 0.1.1

package/dist/cli/main.js

@@ -0,0 +1,3523 @@
+#!/usr/bin/env node
+
+// src/cli/run-cli.ts
+import { Command } from "commander";
+import { stat as stat4 } from "fs/promises";
+import path21 from "path";
+
+// src/chunk/chunker.ts
+import { readFile as readFile3 } from "fs/promises";
+import matter from "gray-matter";
+import path4 from "path";
+
+// src/core/config.ts
+import { readFile, writeFile } from "fs/promises";
+import path from "path";
+import YAML from "yaml";
+var defaultConfig = () => ({
+  workspaceVersion: 1,
+  index: {
+    name: "default",
+    fields: {
+      text: { type: "text", weight: 1 },
+      title: { type: "text", weight: 2 },
+      uri: { type: "keyword" },
+      sourceId: { type: "keyword" },
+      tags: { type: "keyword" },
+      contentType: { type: "keyword" }
+    },
+    chunking: {
+      maxChars: 1800,
+      overlapChars: 200,
+      minChars: 120,
+      splitOnHeadings: true
+    }
+  },
+  rag: {
+    defaultTopK: 12,
+    maxContextChars: 12e3,
+    citationStyle: "markdown"
+  },
+  retrieval: {
+    defaultMode: "lexical",
+    dense: {
+      enabled: false,
+      modelId: "Xenova/all-MiniLM-L6-v2",
+      cacheDir: ".kb/models/huggingface",
+      indexHashTables: 8,
+      indexRandomSeed: 42,
+      chunkTextMode: "title-heading-text"
+    },
+    sparse: {
+      enabled: false,
+      modelId: "opensearch-project/opensearch-neural-sparse-encoding-doc-v3-distill",
+      cacheDir: ".kb/models/huggingface",
+      documentTopTokens: 128,
+      queryEncoding: "tokenizer-token-weights",
+      documentEncoding: "masked-lm-max-log1p-relu",
+      chunkTextMode: "title-heading-text"
+    }
+  },
+  crawler: {
+    defaultUserAgent: "querylight-cli/0.1",
+    obeyRobotsTxt: true,
+    rateLimitMs: 1e3,
+    renderJs: false,
+    retentionDays: 365,
+    fetchArticles: true
+  },
+  limits: {
+    maxFileSizeMb: 50,
+    maxPagesPerSource: 100,
+    maxTotalChunks: 1e5
+  }
+});
+async function writeDefaultConfig(workspacePath, force = false) {
+  const configPath = path.join(workspacePath, "config.yaml");
+  try {
+    if (!force) {
+      await readFile(configPath, "utf8");
+      return;
+    }
+  } catch {
+  }
+  await writeFile(configPath, YAML.stringify(defaultConfig()), "utf8");
+}
+async function loadConfig(workspacePath, configPath) {
+  const resolved = configPath ?? path.join(workspacePath, "config.yaml");
+  const raw = await readFile(resolved, "utf8");
+  const parsed = YAML.parse(raw);
+  const defaults = defaultConfig();
+  return {
+    ...defaults,
+    ...parsed,
+    index: {
+      ...defaults.index,
+      ...parsed.index,
+      fields: {
+        ...defaults.index.fields,
+        ...parsed.index?.fields ?? {}
+      },
+      chunking: {
+        ...defaults.index.chunking,
+        ...parsed.index?.chunking ?? {}
+      }
+    },
+    rag: {
+      ...defaults.rag,
+      ...parsed.rag ?? {}
+    },
+    retrieval: {
+      ...defaults.retrieval,
+      ...parsed.retrieval ?? {},
+      dense: {
+        ...defaults.retrieval.dense,
+        ...parsed.retrieval?.dense ?? {}
+      },
+      sparse: {
+        ...defaults.retrieval.sparse,
+        ...parsed.retrieval?.sparse ?? {}
+      }
+    },
+    crawler: {
+      ...defaults.crawler,
+      ...parsed.crawler ?? {}
+    },
+    limits: {
+      ...defaults.limits,
+      ...parsed.limits ?? {}
+    }
+  };
+}
+
+// src/core/hashing.ts
+import { createHash } from "crypto";
+function sha256(input) {
+  return createHash("sha256").update(input).digest("hex");
+}
+
+// src/core/ids.ts
+function stableId(prefix, ...parts) {
+  return `${prefix}_${sha256(parts.join("::")).slice(0, 16)}`;
+}
+
+// src/core/jsonl.ts
+import { mkdir, readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
+import path2 from "path";
+async function readJsonl(filePath) {
+  try {
+    const raw = await readFile2(filePath, "utf8");
+    return raw.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+}
+async function writeJsonl(filePath, records) {
+  await mkdir(path2.dirname(filePath), { recursive: true });
+  const payload = records.map((record) => JSON.stringify(record)).join("\n");
+  await writeFile2(filePath, payload.length > 0 ? `${payload}
+` : "", "utf8");
+}
+
+// src/chunk/chunk-store.ts
+import path3 from "path";
+function chunksFile(workspacePath) {
+  return path3.join(workspacePath, "chunks", "chunks.jsonl");
+}
+async function loadChunks(workspacePath) {
+  return readJsonl(chunksFile(workspacePath));
+}
+async function saveChunks(workspacePath, chunks) {
+  await writeJsonl(chunksFile(workspacePath), chunks.sort((a, b) => a.id.localeCompare(b.id)));
+}
+
+// src/chunk/chunker.ts
+function splitSections(markdown) {
+  const lines = markdown.split("\n");
+  const sections = [];
+  let headingPath = [];
+  let current = [];
+  const flush = () => {
+    const text = current.join("\n").trim();
+    if (text.length > 0) {
+      sections.push({ headingPath: [...headingPath], text });
+    }
+    current = [];
+  };
+  for (const line of lines) {
+    const match = /^(#{1,6})\s+(.+)$/.exec(line);
+    if (match?.[1] && match[2]) {
+      flush();
+      const level = match[1].length;
+      headingPath = [...headingPath.slice(0, level - 1), match[2].trim()];
+      current.push(line);
+      continue;
+    }
+    current.push(line);
+  }
+  flush();
+  return sections;
+}
+function splitLongSection(text, maxChars, overlapChars) {
+  if (text.length <= maxChars) {
+    return [text];
+  }
+  const chunks = [];
+  let start = 0;
+  while (start < text.length) {
+    const hardEnd = Math.min(text.length, start + maxChars);
+    let sliceEnd = hardEnd;
+    const window = text.slice(start, hardEnd);
+    const paragraphBreak = window.lastIndexOf("\n\n");
+    if (paragraphBreak > maxChars / 2 && hardEnd < text.length) {
+      const candidateEnd = start + paragraphBreak;
+      if (candidateEnd - start > overlapChars) {
+        sliceEnd = candidateEnd;
+      }
+    }
+    const slice = text.slice(start, sliceEnd).trim();
+    if (slice.length === 0) {
+      start = hardEnd;
+      continue;
+    }
+    chunks.push(slice);
+    const nextStart = sliceEnd - overlapChars;
+    start = nextStart > start ? nextStart : hardEnd;
+  }
+  return chunks.filter((chunk) => chunk.length > 0);
+}
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function buildChunksForDocument(document, markdown, config, prior = /* @__PURE__ */ new Map(), seenAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  const parsed = matter(markdown);
+  const sections = splitSections(parsed.content);
+  const usefulSections = sections.length > 0 ? sections : [{ headingPath: [document.title], text: parsed.content }];
+  const chunks = [];
+  for (const section of usefulSections) {
+    const pieces = splitLongSection(section.text, config.index.chunking.maxChars, config.index.chunking.overlapChars);
+    for (const piece of pieces) {
+      if (piece.trim().length < Math.min(40, config.index.chunking.minChars) && pieces.length === 1) {
+        continue;
+      }
+      const text = piece.trim();
+      const id = stableId("chunk", document.id, section.headingPath.join(" > "), text);
+      const priorChunk = prior.get(id);
+      const contentHash = sha256(text);
+      chunks.push({
+        id,
+        documentId: document.id,
+        sourceId: document.sourceId,
+        title: document.title,
+        uri: document.uri,
+        headingPath: section.headingPath,
+        text,
+        tokenEstimate: estimateTokens(text),
+        contentHash,
+        metadata: document.metadata,
+        firstSeenAt: priorChunk?.firstSeenAt ?? document.firstSeenAt,
+        lastSeenAt: seenAt,
+        lastChangedAt: priorChunk?.contentHash === contentHash ? priorChunk.lastChangedAt : document.lastChangedAt
+      });
+    }
+  }
+  return chunks;
+}
+async function chunkDocuments({
+  workspacePath,
+  sourceId,
+  documentId
+}) {
+  const config = await loadConfig(workspacePath);
+  const documents = await readJsonl(path4.join(workspacePath, "documents", "documents.jsonl"));
+  const filtered = documents.filter((document) => (!sourceId || document.sourceId === sourceId) && (!documentId || document.id === documentId));
+  const targetedDocumentIds = new Set(filtered.map((document) => document.id));
+  const existingChunks = await loadChunks(workspacePath);
+  const prior = new Map(existingChunks.map((chunk) => [chunk.id, chunk]));
+  const nextChunks = new Map(
+    existingChunks.filter((chunk) => !targetedDocumentIds.has(chunk.documentId)).map((chunk) => [chunk.id, chunk])
+  );
+  for (const document of filtered) {
+    const raw = await readFile3(document.normalizedPath, "utf8");
+    for (const chunk of buildChunksForDocument(document, raw, config, prior)) {
+      nextChunks.set(chunk.id, chunk);
+    }
+  }
+  await saveChunks(workspacePath, [...nextChunks.values()]);
+  return { chunksWritten: nextChunks.size };
+}
+
+// src/core/constants.ts
+var PACKAGE_VERSION = "0.1.0";
+var DEFAULT_WORKSPACE = ".kb";
+
+// src/core/errors.ts
+var CliError = class extends Error {
+  constructor(message, code, exitCode, details) {
+    super(message);
+    this.code = code;
+    this.exitCode = exitCode;
+    this.details = details;
+    this.name = "CliError";
+  }
+  code;
+  exitCode;
+  details;
+};
+
+// src/core/workspace.ts
+import { mkdir as mkdir2, stat } from "fs/promises";
+import path5 from "path";
+var DIRS = [
+  "sources",
+  "documents",
+  "chunks",
+  "raw",
+  "normalized",
+  "indexes",
+  "vectors",
+  "models",
+  "models/huggingface",
+  "runs",
+  "logs"
+];
+async function ensureWorkspace({
+  workspacePath,
+  force = false
+}) {
+  const resolved = path5.resolve(workspacePath);
+  await mkdir2(resolved, { recursive: true });
+  for (const dir of DIRS) {
+    await mkdir2(path5.join(resolved, dir), { recursive: true });
+  }
+  await writeDefaultConfig(resolved, force);
+  return { workspacePath: resolved };
+}
+async function assertWorkspaceExists(workspacePath) {
+  const resolved = path5.resolve(workspacePath);
+  try {
+    const info = await stat(resolved);
+    if (!info.isDirectory()) {
+      throw new CliError(`workspace is not a directory: ${resolved}`, "WORKSPACE_ERROR", 3 /* WorkspaceError */);
+    }
+    await stat(path5.join(resolved, "config.yaml"));
+    return resolved;
+  } catch (error) {
+    if (error instanceof CliError) {
+      throw error;
+    }
+    throw new CliError(`workspace does not exist or is invalid: ${resolved}`, "WORKSPACE_ERROR", 3 /* WorkspaceError */);
+  }
+}
+
+// src/index/querylight-indexer.ts
+import { Analyzer, DocumentIndex, KeywordTokenizer, LowerCaseTextFilter, RankingAlgorithm, TextFieldIndex } from "@tryformation/querylight-ts";
+import path11 from "path";
+
+// src/vector/dense.ts
+import { VectorFieldIndex, createSeededRandom } from "@tryformation/querylight-ts";
+import { mkdir as mkdir4 } from "fs/promises";
+import path8 from "path";
+
+// src/vector/runtime.ts
+import path6 from "path";
+import { fileURLToPath } from "url";
+import { execFile, execFileSync } from "child_process";
+
+// src/core/files.ts
+import { stat as stat2 } from "fs/promises";
+async function fileExists(filePath) {
+  try {
+    await stat2(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// src/vector/runtime.ts
+function resolveCacheDir(workspacePath, configuredPath) {
+  return path6.isAbsolute(configuredPath) ? configuredPath : path6.resolve(workspacePath, configuredPath.replace(/^\.kb\//, ""));
+}
+function packageRootFromImportMeta(importMetaUrl) {
+  return path6.resolve(path6.dirname(fileURLToPath(importMetaUrl)), "..");
+}
+async function sparseScriptPath(importMetaUrl) {
+  const base = packageRootFromImportMeta(importMetaUrl);
+  const candidates = [
+    path6.join(base, "scripts", "sparse-encode.py"),
+    path6.join(base, "..", "scripts", "sparse-encode.py")
+  ];
+  for (const candidate of candidates) {
+    if (await fileExists(candidate)) {
+      return path6.resolve(candidate);
+    }
+  }
+  throw new Error(`sparse helper script not found; checked ${candidates.join(", ")}`);
+}
+async function ensureUvAvailable() {
+  await new Promise((resolve2, reject) => {
+    execFile("uv", ["--version"], (error) => error ? reject(error) : resolve2());
+  });
+}
+async function runSparsePython({
+  workspacePath,
+  config,
+  payload,
+  importMetaUrl
+}) {
+  const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
+  const scriptPath = await sparseScriptPath(importMetaUrl);
+  return execFileSync(
+    "uv",
+    [
+      "run",
+      "--with",
+      "torch",
+      "--with",
+      "transformers",
+      "--with",
+      "huggingface_hub",
+      "python",
+      scriptPath
+    ],
+    {
+      encoding: "utf8",
+      maxBuffer: 1024 * 1024 * 1024,
+      input: JSON.stringify(payload),
+      env: {
+        ...process.env,
+        HF_HOME: cacheDir
+      }
+    }
+  );
+}
+async function getDenseTransformersRuntime(cacheDir) {
+  const transformers = await import("@huggingface/transformers");
+  transformers.env.cacheDir = cacheDir;
+  transformers.env.allowLocalModels = true;
+  return {
+    env: transformers.env,
+    pipeline: transformers.pipeline
+  };
+}
+
+// src/vector/store.ts
+import { mkdir as mkdir3, readFile as readFile4, writeFile as writeFile3 } from "fs/promises";
+import path7 from "path";
+function vectorsDir(workspacePath) {
+  return path7.join(workspacePath, "vectors");
+}
+function modelsDir(workspacePath) {
+  return path7.join(workspacePath, "models");
+}
+function denseVectorPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "dense.latest.json");
+}
+function denseMetaPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+}
+function sparseVectorPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.json");
+}
+function sparseMetaPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.meta.json");
+}
+function densePullMarker(workspacePath) {
+  return path7.join(modelsDir(workspacePath), "dense.pulled.json");
+}
+function sparsePullMarker(workspacePath) {
+  return path7.join(modelsDir(workspacePath), "sparse.pulled.json");
+}
+async function writeDensePayload(workspacePath, payload) {
+  await mkdir3(vectorsDir(workspacePath), { recursive: true });
+  await writeFile3(denseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
+  await writeFile3(denseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+}
+async function readDensePayload(workspacePath) {
+  return JSON.parse(await readFile4(denseVectorPath(workspacePath), "utf8"));
+}
+async function writeSparsePayload(workspacePath, payload) {
+  await mkdir3(vectorsDir(workspacePath), { recursive: true });
+  await writeFile3(sparseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
+  await writeFile3(sparseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+}
+async function readSparsePayload(workspacePath) {
+  return JSON.parse(await readFile4(sparseVectorPath(workspacePath), "utf8"));
+}
+async function writeDensePullMarker(workspacePath, value) {
+  await mkdir3(modelsDir(workspacePath), { recursive: true });
+  await writeFile3(densePullMarker(workspacePath), JSON.stringify(value, null, 2), "utf8");
+}
+async function writeSparsePullMarker(workspacePath, value) {
+  await mkdir3(modelsDir(workspacePath), { recursive: true });
+  await writeFile3(sparsePullMarker(workspacePath), JSON.stringify(value, null, 2), "utf8");
+}
+async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
+  const denseCacheDir = resolveCacheDir(workspacePath, dense.cacheDir);
+  const sparseCacheDir = resolveCacheDir(workspacePath, sparse.cacheDir);
+  return {
+    dense: {
+      configured: dense.enabled,
+      modelId: dense.modelId,
+      cacheDir: denseCacheDir,
+      available: await fileExists(densePullMarker(workspacePath)),
+      artifactExists: await fileExists(denseVectorPath(workspacePath))
+    },
+    sparse: {
+      configured: sparse.enabled,
+      modelId: sparse.modelId,
+      cacheDir: sparseCacheDir,
+      uvAvailable,
+      available: await fileExists(sparsePullMarker(workspacePath)),
+      artifactExists: await fileExists(sparseVectorPath(workspacePath))
+    }
+  };
+}
+
+// src/vector/text.ts
+function createDenseChunkText(chunk) {
+  return [chunk.title, ...chunk.headingPath, chunk.text].filter(Boolean).join("\n\n");
+}
+function createSparseChunkText(chunk) {
+  return [chunk.title, ...chunk.headingPath, chunk.text].filter(Boolean).join("\n\n");
+}
+
+// src/vector/dense.ts
+var denseEmbedderFactory = null;
+async function createEmbedder(cacheDir, modelId) {
+  if (denseEmbedderFactory) {
+    return denseEmbedderFactory(cacheDir, modelId);
+  }
+  const runtime = await getDenseTransformersRuntime(cacheDir);
+  const extractor = await runtime.pipeline("feature-extraction", modelId);
+  return async (text) => {
+    const output = await extractor(text, { pooling: "mean", normalize: true });
+    return output.tolist()[0];
+  };
+}
+async function pullDenseModel(workspacePath, config) {
+  const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
+  await mkdir4(cacheDir, { recursive: true });
+  const embed = await createEmbedder(cacheDir, config.modelId);
+  await embed("warm dense model cache");
+}
+async function buildDenseVectors({
+  workspacePath,
+  config
+}) {
+  const chunks = await readJsonl(path8.join(workspacePath, "chunks", "chunks.jsonl"));
+  const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
+  await mkdir4(cacheDir, { recursive: true });
+  const embed = await createEmbedder(cacheDir, config.modelId);
+  const records = [];
+  let dimensions = 0;
+  for (const chunk of chunks) {
+    const embedding = await embed(createDenseChunkText(chunk));
+    dimensions ||= embedding.length;
+    records.push({
+      chunkId: chunk.id,
+      documentId: chunk.documentId,
+      sourceId: chunk.sourceId,
+      title: chunk.title,
+      uri: chunk.uri,
+      headingPath: chunk.headingPath,
+      text: chunk.text,
+      embedding
+    });
+  }
+  const index = new VectorFieldIndex({
+    numHashTables: config.indexHashTables,
+    dimensions,
+    random: createSeededRandom(config.indexRandomSeed)
+  });
+  for (const record of records) {
+    index.insert(record.chunkId, [record.embedding]);
+  }
+  const metadata = {
+    createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+    modelId: config.modelId,
+    dimensions,
+    hashTables: config.indexHashTables,
+    randomSeed: config.indexRandomSeed,
+    chunkCount: records.length,
+    indexHash: sha256(JSON.stringify(index.indexState))
+  };
+  const payload = {
+    metadata,
+    indexState: index.indexState,
+    chunks: records
+  };
+  await writeDensePayload(workspacePath, payload);
+  return payload;
+}
+async function denseQuery({
+  workspacePath,
+  config,
+  query,
+  topK
+}) {
+  const payload = await readDensePayload(workspacePath);
+  const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
+  const embed = await createEmbedder(cacheDir, config.modelId);
+  const vector = await embed(query);
+  const index = new VectorFieldIndex({
+    numHashTables: payload.metadata.hashTables,
+    dimensions: payload.metadata.dimensions,
+    random: createSeededRandom(payload.metadata.randomSeed)
+  }).loadState(payload.indexState);
+  return index.query(vector, topK);
+}
+
+// src/vector/sparse.ts
+import { SparseVectorFieldIndex } from "@tryformation/querylight-ts";
+import { mkdir as mkdir5 } from "fs/promises";
+import path9 from "path";
+var sparseQueryEncoderFactory = null;
+var sparseDocumentBuilderFactory = null;
+function buildSparseQueryVector(tokenIds, tokenWeights) {
+  const sparseVector = {};
+  for (const tokenId of new Set(tokenIds)) {
+    const weight = tokenWeights[tokenId] ?? 0;
+    if (weight > 0) {
+      sparseVector[String(tokenId)] = weight;
+    }
+  }
+  return sparseVector;
+}
+function normalizeTokenIds(value) {
+  if (value && typeof value === "object" && "data" in value) {
+    const data = value.data;
+    if (Array.isArray(data)) {
+      return data.map(Number).filter(Number.isFinite);
+    }
+    if (ArrayBuffer.isView(data)) {
+      return Array.from(data, Number).filter(Number.isFinite);
+    }
+  }
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  if (value.length === 0) {
+    return [];
+  }
+  if (Array.isArray(value[0])) {
+    return value[0].map(Number).filter(Number.isFinite);
+  }
+  return value.map(Number).filter(Number.isFinite);
+}
+async function createSparseQueryEncoder(cacheDir, modelId, queryTokenWeights) {
+  if (sparseQueryEncoderFactory) {
+    return sparseQueryEncoderFactory(cacheDir, modelId, queryTokenWeights);
+  }
+  const runtime = await getDenseTransformersRuntime(cacheDir);
+  const { AutoTokenizer } = await import("@huggingface/transformers");
+  runtime.env.cacheDir = cacheDir;
+  const tokenizer = await AutoTokenizer.from_pretrained(modelId);
+  return async (text) => {
+    const features = await tokenizer([text], {
+      truncation: true,
+      return_attention_mask: false,
+      return_token_type_ids: false
+    });
+    return buildSparseQueryVector(normalizeTokenIds(features.input_ids), queryTokenWeights);
+  };
+}
+async function pullSparseModel(workspacePath, config) {
+  await ensureUvAvailable();
+  const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
+  await mkdir5(cacheDir, { recursive: true });
+  await runSparsePython({
+    workspacePath,
+    config,
+    importMetaUrl: import.meta.url,
+    payload: {
+      action: "download_only",
+      model_id: config.modelId
+    }
+  });
+}
+async function buildSparseDocuments(workspacePath, config, chunks) {
+  if (sparseDocumentBuilderFactory) {
+    return sparseDocumentBuilderFactory(workspacePath, config, chunks);
+  }
+  await ensureUvAvailable();
+  const output = JSON.parse(await runSparsePython({
+    workspacePath,
+    config,
+    importMetaUrl: import.meta.url,
+    payload: {
+      action: "encode_documents",
+      model_id: config.modelId,
+      top_tokens: config.documentTopTokens,
+      documents: chunks.map((chunk) => ({
+        chunkId: chunk.id,
+        text: createSparseChunkText(chunk)
+      }))
+    }
+  }));
+  const byId = new Map(output.documents.map((document) => [document.chunkId, document.vector]));
+  return {
+    queryTokenWeights: output.query_token_weights,
+    vocabularySize: output.vocabularySize,
+    chunks: chunks.map((chunk) => ({
+      chunkId: chunk.id,
+      documentId: chunk.documentId,
+      sourceId: chunk.sourceId,
+      title: chunk.title,
+      uri: chunk.uri,
+      headingPath: chunk.headingPath,
+      text: chunk.text,
+      vector: byId.get(chunk.id) ?? {}
+    }))
+  };
+}
+async function buildSparseVectors({
+  workspacePath,
+  config
+}) {
+  const chunks = await readJsonl(path9.join(workspacePath, "chunks", "chunks.jsonl"));
+  const built = await buildSparseDocuments(workspacePath, config, chunks);
+  const index = new SparseVectorFieldIndex();
+  for (const record of built.chunks) {
+    index.insert(record.chunkId, [record.vector]);
+  }
+  const metadata = {
+    createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+    modelId: config.modelId,
+    vocabularySize: built.vocabularySize,
+    documentTopTokens: config.documentTopTokens,
+    queryEncoding: config.queryEncoding,
+    documentEncoding: config.documentEncoding,
+    chunkCount: built.chunks.length,
+    indexHash: sha256(JSON.stringify(index.indexState))
+  };
+  const payload = {
+    metadata,
+    indexState: index.indexState,
+    chunks: built.chunks,
+    queryTokenWeights: built.queryTokenWeights
+  };
+  await writeSparsePayload(workspacePath, payload);
+  return payload;
+}
+async function sparseQuery({
+  workspacePath,
+  config,
+  query,
+  topK
+}) {
+  const payload = await readSparsePayload(workspacePath);
+  const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
+  const encode = await createSparseQueryEncoder(cacheDir, config.modelId, payload.queryTokenWeights);
+  const vector = await encode(query);
+  const index = new SparseVectorFieldIndex().loadState(payload.indexState);
+  return index.query(vector, topK);
+}
+
+// src/vector/service.ts
+function resolveModelPullPlan({
+  pullDenseFlag,
+  pullSparseFlag,
+  uvAvailable
+}) {
+  if (pullDenseFlag || pullSparseFlag) {
+    return {
+      pullDense: pullDenseFlag,
+      pullSparse: pullSparseFlag
+    };
+  }
+  return {
+    pullDense: true,
+    pullSparse: uvAvailable
+  };
+}
+async function buildVectorArtifacts({
+  workspacePath,
+  config,
+  denseOverride,
+  sparseOverride,
+  buildAvailableModels = false
+}) {
+  const modelStatus = buildAvailableModels ? await buildModelStatus(workspacePath, config.retrieval.dense, config.retrieval.sparse, await (async () => {
+    try {
+      await ensureUvAvailable();
+      return true;
+    } catch {
+      return false;
+    }
+  })()) : null;
+  const denseEnabled = denseOverride ?? (buildAvailableModels ? config.retrieval.dense.enabled || Boolean(modelStatus?.dense.available) : config.retrieval.dense.enabled);
+  const sparseEnabled = sparseOverride ?? (buildAvailableModels ? (config.retrieval.sparse.enabled || Boolean(modelStatus?.sparse.available)) && Boolean(modelStatus?.sparse.uvAvailable) : config.retrieval.sparse.enabled);
+  const result2 = {};
+  if (denseEnabled) {
+    result2.dense = await buildDenseVectors({ workspacePath, config: config.retrieval.dense });
+  }
+  if (sparseEnabled) {
+    result2.sparse = await buildSparseVectors({ workspacePath, config: config.retrieval.sparse });
+  }
+  return result2;
+}
+async function pullModels({
+  workspacePath,
+  config,
+  pullDense,
+  pullSparse
+}) {
+  if (pullDense) {
+    await pullDenseModel(workspacePath, config.retrieval.dense);
+    await writeDensePullMarker(workspacePath, {
+      pulledAt: (/* @__PURE__ */ new Date()).toISOString(),
+      modelId: config.retrieval.dense.modelId
+    });
+  }
+  if (pullSparse) {
+    await pullSparseModel(workspacePath, config.retrieval.sparse);
+    await writeSparsePullMarker(workspacePath, {
+      pulledAt: (/* @__PURE__ */ new Date()).toISOString(),
+      modelId: config.retrieval.sparse.modelId
+    });
+  }
+}
+async function getModelStatus(workspacePath, config) {
+  let uvAvailable = false;
+  try {
+    await ensureUvAvailable();
+    uvAvailable = true;
+  } catch {
+    uvAvailable = false;
+  }
+  return buildModelStatus(workspacePath, config.retrieval.dense, config.retrieval.sparse, uvAvailable);
+}
+
+// src/index/index-store.ts
+import { readFile as readFile5, writeFile as writeFile4 } from "fs/promises";
+import path10 from "path";
+async function writeIndexArtifacts({
+  workspacePath,
+  indexState,
+  metadata
+}) {
+  const stamp = metadata.createdAt.replace(/[:.]/g, "-");
+  const indexPath = path10.join(workspacePath, "indexes", `${stamp}.json`);
+  const metaPath = path10.join(workspacePath, "indexes", `${stamp}.meta.json`);
+  const latestIndexPath = path10.join(workspacePath, "indexes", "latest.json");
+  const latestMetaPath = path10.join(workspacePath, "indexes", "latest.meta.json");
+  const indexPayload = JSON.stringify(indexState, null, 2);
+  const metaPayload = JSON.stringify(metadata, null, 2);
+  await writeFile4(indexPath, indexPayload, "utf8");
+  await writeFile4(metaPath, metaPayload, "utf8");
+  await writeFile4(latestIndexPath, indexPayload, "utf8");
+  await writeFile4(latestMetaPath, metaPayload, "utf8");
+  return { indexPath: latestIndexPath, metadataPath: latestMetaPath };
+}
+async function readLatestIndexState(workspacePath) {
+  return JSON.parse(await readFile5(path10.join(workspacePath, "indexes", "latest.json"), "utf8"));
+}
+async function readLatestIndexMetadata(workspacePath) {
+  return JSON.parse(await readFile5(path10.join(workspacePath, "indexes", "latest.meta.json"), "utf8"));
+}
+
+// src/index/querylight-indexer.ts
+function keywordFieldIndex() {
+  const analyzer = new Analyzer([new LowerCaseTextFilter()], new KeywordTokenizer());
+  return new TextFieldIndex(analyzer, analyzer, RankingAlgorithm.BM25);
+}
+function createIndexMapping(extraFields = []) {
+  const lexical = new TextFieldIndex(void 0, void 0, RankingAlgorithm.BM25);
+  const mapping = {
+    text: lexical,
+    title: new TextFieldIndex(void 0, void 0, RankingAlgorithm.BM25),
+    uri: keywordFieldIndex(),
+    sourceId: keywordFieldIndex(),
+    tags: keywordFieldIndex(),
+    sourceType: keywordFieldIndex()
+  };
+  for (const field of extraFields) {
+    mapping[field] = keywordFieldIndex();
+  }
+  return mapping;
+}
+function flattenMetadata(metadata) {
+  const flattened = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    if (value == null) {
+      continue;
+    }
+    const field = `metadata.${key}`;
+    if (Array.isArray(value)) {
+      flattened[field] = value.map((item) => String(item).toLowerCase());
+    } else {
+      flattened[field] = [String(value).toLowerCase()];
+    }
+  }
+  return flattened;
+}
+async function buildIndex({
+  workspacePath,
+  denseOverride,
+  sparseOverride,
+  buildAvailableModels = false
+}) {
+  const config = await loadConfig(workspacePath);
+  const chunks = await readJsonl(path11.join(workspacePath, "chunks", "chunks.jsonl"));
+  const documents = await readJsonl(path11.join(workspacePath, "documents", "documents.jsonl"));
+  const sources = await readJsonl(path11.join(workspacePath, "sources", "sources.jsonl"));
+  const metadataFields = [...new Set(chunks.flatMap((chunk) => Object.keys(chunk.metadata).map((key) => `metadata.${key}`)))];
+  const index = new DocumentIndex(createIndexMapping(metadataFields));
+  for (const chunk of chunks) {
+    index.index({
+      id: chunk.id,
+      fields: {
+        text: [chunk.text],
+        title: [chunk.title],
+        uri: [chunk.uri.toLowerCase()],
+        sourceId: [chunk.sourceId.toLowerCase()],
+        tags: Array.isArray(chunk.metadata.tags) ? chunk.metadata.tags.map((tag) => String(tag).toLowerCase()) : [],
+        sourceType: [String(chunk.metadata.sourceType ?? "").toLowerCase()],
+        ...flattenMetadata(chunk.metadata)
+      }
+    });
+  }
+  const createdAt = (/* @__PURE__ */ new Date()).toISOString();
+  const metadata = {
+    id: `index_${createdAt.replace(/[:.]/g, "-")}`,
+    createdAt,
+    querylightVersion: "0.10.0",
+    kbVersion: "0.1.0",
+    documentCount: documents.length,
+    chunkCount: chunks.length,
+    sourceCount: sources.length,
+    fields: Object.keys(index.mapping),
+    indexHash: sha256(JSON.stringify(index.indexState))
+  };
+  const artifacts = await writeIndexArtifacts({ workspacePath, indexState: index.indexState, metadata });
+  const vectors = await buildVectorArtifacts({
+    workspacePath,
+    config,
+    denseOverride,
+    sparseOverride,
+    buildAvailableModels
+  });
+  return {
+    metadata,
+    indexPath: artifacts.indexPath,
+    denseBuilt: Boolean(vectors.dense),
+    sparseBuilt: Boolean(vectors.sparse)
+  };
+}
+
+// src/ingest/ingest-service.ts
+import path17 from "path";
+
+// src/core/runs.ts
+import path12 from "path";
+async function writeRun(workspacePath, run) {
+  await writeJsonl(path12.join(workspacePath, "runs", `${run.id}.json`), [run]);
+}
+async function listRuns(workspacePath) {
+  const fs = await import("fs/promises");
+  const dir = path12.join(workspacePath, "runs");
+  try {
+    const entries = await fs.readdir(dir);
+    const records = await Promise.all(entries.filter((name) => name.endsWith(".json")).map(async (name) => {
+      const runs = await readJsonl(path12.join(dir, name));
+      return runs[0];
+    }));
+    return records.filter((record) => record != null).sort((a, b) => a.createdAt.localeCompare(b.createdAt));
+  } catch {
+    return [];
+  }
+}
+
+// src/sources/source-store.ts
+import path13 from "path";
+var sourcesFile = (workspacePath) => path13.join(workspacePath, "sources", "sources.jsonl");
+async function listSources(workspacePath) {
+  return readJsonl(sourcesFile(workspacePath));
+}
+async function addSource(workspacePath, source) {
+  const existing = await listSources(workspacePath);
+  if (existing.some((candidate) => candidate.uri === source.uri)) {
+    throw new CliError(`duplicate source URI: ${source.uri}`, "DUPLICATE_SOURCE", 4 /* SourceError */);
+  }
+  const id = source.id ?? stableId("src", source.type, source.uri);
+  const stored = { ...source, id };
+  existing.push(stored);
+  await writeJsonl(sourcesFile(workspacePath), existing);
+  return stored;
+}
+async function updateSource(workspacePath, sourceId, patch) {
+  const sources = await listSources(workspacePath);
+  const index = sources.findIndex((source) => source.id === sourceId);
+  if (index < 0) {
+    throw new CliError(`source not found: ${sourceId}`, "SOURCE_NOT_FOUND", 4 /* SourceError */);
+  }
+  const current = sources[index];
+  const updated = {
+    ...current,
+    ...patch,
+    id: sourceId,
+    metadata: patch.metadata ? { ...current.metadata, ...patch.metadata } : current.metadata,
+    crawl: patch.crawl ? {
+      ...current.crawl ?? {},
+      ...patch.crawl
+    } : current.crawl
+  };
+  sources[index] = updated;
+  await writeJsonl(sourcesFile(workspacePath), sources);
+  return updated;
+}
+async function removeSource(workspacePath, sourceId) {
+  const sources = await listSources(workspacePath);
+  const filtered = sources.filter((source) => source.id !== sourceId);
+  if (filtered.length === sources.length) {
+    throw new CliError(`source not found: ${sourceId}`, "SOURCE_NOT_FOUND", 4 /* SourceError */);
+  }
+  await writeJsonl(sourcesFile(workspacePath), filtered);
+}
+
+// src/ingest/document-utils.ts
+import { mkdir as mkdir6, rm, writeFile as writeFile5 } from "fs/promises";
+import path14 from "path";
+
+// src/normalize/normalize-markdown.ts
+import matter2 from "gray-matter";
+function normalizeWhitespace(text) {
+  return text.replace(/\r\n/g, "\n").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+function withFrontmatter(metadata, body) {
+  return matter2.stringify(normalizeWhitespace(body), metadata);
+}
+
+// src/ingest/document-utils.ts
+function asMetadataValue(value) {
+  return value === void 0 ? void 0 : value;
+}
+function buildDocumentMetadata({
+  source,
+  sourceUri,
+  publicationDate,
+  crawledAt,
+  indexedAt,
+  extra = {}
+}) {
+  const merged = {
+    ...source.metadata,
+    ...extra,
+    tags: source.tags,
+    sourceType: source.type,
+    sourceUri,
+    publicationDate: publicationDate ?? null,
+    crawledAt,
+    indexedAt
+  };
+  const filtered = Object.fromEntries(
+    Object.entries(merged).filter(([, value]) => asMetadataValue(value) !== void 0)
+  );
+  return filtered;
+}
+async function writeNormalizedDocument({
+  documentId,
+  sourceId,
+  title,
+  uri,
+  sourceUri,
+  publicationDate,
+  crawledAt,
+  indexedAt,
+  contentHash,
+  lastChangedAt,
+  normalizedPath,
+  markdown
+}) {
+  await mkdir6(path14.dirname(normalizedPath), { recursive: true });
+  await writeFile5(
+    normalizedPath,
+    withFrontmatter(
+      {
+        documentId,
+        sourceId,
+        title,
+        uri,
+        sourceUri,
+        publicationDate: publicationDate ?? null,
+        crawledAt,
+        indexedAt,
+        contentHash,
+        lastChangedAt
+      },
+      markdown
+    ),
+    "utf8"
+  );
+}
+async function deleteDocumentArtifacts(document) {
+  await Promise.all([
+    document.rawPath ? rm(document.rawPath, { force: true }) : Promise.resolve(),
+    rm(document.normalizedPath, { force: true })
+  ]);
+}
+
+// src/ingest/adapters/directory-adapter.ts
+import fg from "fast-glob";
+import path15 from "path";
+async function listDirectoryFiles(source) {
+  const include = source.crawl?.includePatterns?.length ? source.crawl.includePatterns : ["**/*.md", "**/*.txt", "**/*.html", "**/*.htm", "**/*.pdf", "**/*.docx"];
+  const exclude = source.crawl?.excludePatterns ?? [];
+  const matches = await fg(include, {
+    cwd: source.uri,
+    absolute: true,
+    onlyFiles: true,
+    dot: false,
+    unique: true,
+    ignore: exclude,
+    followSymbolicLinks: false
+  });
+  return matches.map((match) => path15.resolve(match)).sort();
+}
+
+// src/ingest/adapters/file-adapter.ts
+import { basename, extname, resolve } from "path";
+import { mkdir as mkdir7, readFile as readFile9, stat as stat3, writeFile as writeFile6 } from "fs/promises";
+
+// src/ingest/extractors/docx-extractor.ts
+import mammoth from "mammoth";
+async function extractDocx(filePath) {
+  const result2 = await mammoth.extractRawText({ path: filePath });
+  return result2.value;
+}
+
+// src/ingest/extractors/html-extractor.ts
+import { load } from "cheerio";
+import TurndownService from "turndown";
+
+// src/normalize/boilerplate.ts
+function stripBoilerplate(html) {
+  return html.replace(/<nav[\s\S]*?<\/nav>/gi, "").replace(/<footer[\s\S]*?<\/footer>/gi, "").replace(/cookie notice/gi, "");
+}
+
+// src/ingest/extractors/html-extractor.ts
+var turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+function cleanText(value) {
+  return value.replace(/\s+/g, " ").trim();
+}
+function chooseMeaningfulTitle($, fallbackTitle) {
+  const candidates = [
+    cleanText($("meta[property='og:title']").attr("content") ?? ""),
+    cleanText($("meta[name='twitter:title']").attr("content") ?? ""),
+    cleanText($("h1").first().text()),
+    cleanText($("title").first().text()),
+    fallbackTitle
+  ].filter(Boolean);
+  return candidates[0] ?? fallbackTitle;
+}
+turndown.addRule("docCard", {
+  filter(node) {
+    return node.nodeName === "A" && typeof node.getAttribute === "function" && (node.getAttribute("class") ?? "").split(/\s+/).includes("doc-card");
+  },
+  replacement(_content, node) {
+    const element = node;
+    const href = cleanText(element.getAttribute("href") ?? "");
+    const title = cleanText(element.querySelector("h3")?.textContent ?? "");
+    const summary = cleanText(element.querySelector("p")?.textContent ?? "");
+    const section = cleanText(element.querySelector("span")?.textContent ?? "");
+    const parts = [
+      title ? `### ${title}` : "",
+      summary,
+      section,
+      href
+    ].filter(Boolean);
+    return `
+
+${parts.join("\n\n")}
+
+`;
+  }
+});
+function extractHtmlToMarkdown(html) {
+  const cleaned = stripBoilerplate(html);
+  const $ = load(cleaned);
+  const fallbackTitle = cleanText($("title").first().text()) || "Untitled";
+  const title = chooseMeaningfulTitle($, fallbackTitle);
+  const root = $("main").first().html() ?? $.root().html() ?? cleaned;
+  return {
+    markdown: turndown.turndown(root),
+    title
+  };
+}
+function parseDateCandidate(value) {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return null;
+  }
+  const date = new Date(trimmed);
+  return Number.isNaN(date.getTime()) ? null : date.toISOString();
+}
+function extractPublicationDateFromHtml(html) {
+  const $ = load(html);
+  const candidates = [
+    $("meta[property='article:published_time']").attr("content"),
+    $("meta[property='og:published_time']").attr("content"),
+    $("meta[name='pubdate']").attr("content"),
+    $("meta[name='publish-date']").attr("content"),
+    $("meta[name='article:published_time']").attr("content"),
+    $("meta[name='date']").attr("content"),
+    $("time[datetime]").first().attr("datetime")
+  ].filter((value) => Boolean(value?.trim()));
+  for (const candidate of candidates) {
+    const parsed = parseDateCandidate(candidate);
+    if (parsed) {
+      return parsed;
+    }
+  }
+  let jsonLdDate = null;
+  $('script[type="application/ld+json"]').each((_, element) => {
+    if (jsonLdDate) {
+      return false;
+    }
+    try {
+      const raw = $(element).text();
+      const parsed = JSON.parse(raw);
+      const queue = Array.isArray(parsed) ? [...parsed] : [parsed];
+      while (queue.length > 0) {
+        const next = queue.shift();
+        if (!next || typeof next !== "object") {
+          continue;
+        }
+        const record = next;
+        for (const key of ["datePublished", "dateCreated", "dateModified"]) {
+          if (typeof record[key] === "string") {
+            const normalized = parseDateCandidate(record[key]);
+            if (normalized) {
+              jsonLdDate = normalized;
+              return false;
+            }
+          }
+        }
+        if (Array.isArray(record["@graph"])) {
+          queue.push(...record["@graph"]);
+        }
+      }
+    } catch (error) {
+      void error;
+    }
+    return void 0;
+  });
+  return jsonLdDate;
+}
+
+// src/ingest/extractors/markdown-extractor.ts
+import { readFile as readFile6 } from "fs/promises";
+async function extractMarkdown(filePath) {
+  return readFile6(filePath, "utf8");
+}
+
+// src/ingest/extractors/pdf-extractor.ts
+import { readFile as readFile7 } from "fs/promises";
+import { PDFParse } from "pdf-parse";
+async function extractPdf(filePath) {
+  const buffer = await readFile7(filePath);
+  const parser = new PDFParse({ data: buffer });
+  try {
+    const parsed = await parser.getText();
+    return parsed.text;
+  } finally {
+    await parser.destroy();
+  }
+}
+
+// src/ingest/extractors/text-extractor.ts
+import { readFile as readFile8 } from "fs/promises";
+async function extractText(filePath) {
+  return readFile8(filePath, "utf8");
+}
+
+// src/ingest/adapters/file-adapter.ts
+function mimeTypeFor(filePath) {
+  const ext = extname(filePath).toLowerCase();
+  switch (ext) {
+    case ".md":
+      return "text/markdown";
+    case ".txt":
+      return "text/plain";
+    case ".html":
+    case ".htm":
+      return "text/html";
+    case ".pdf":
+      return "application/pdf";
+    case ".docx":
+      return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+    default:
+      return "application/octet-stream";
+  }
+}
+async function extractFileContent(filePath, mimeType) {
+  if (mimeType === "text/markdown") {
+    const markdown = await extractMarkdown(filePath);
+    const title = markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? basename(filePath);
+    return { title, markdown, raw: markdown };
+  }
+  if (mimeType === "text/plain") {
+    const text = await extractText(filePath);
+    return { title: basename(filePath), markdown: `# ${basename(filePath)}
+
+${text}`, raw: text };
+  }
+  if (mimeType === "text/html") {
+    const raw = await readFile9(filePath, "utf8");
+    const extracted = extractHtmlToMarkdown(raw);
+    return { title: extracted.title, markdown: `# ${extracted.title}
+
+${extracted.markdown}`, raw };
+  }
+  if (mimeType === "application/pdf") {
+    const text = await extractPdf(filePath);
+    return { title: basename(filePath), markdown: `# ${basename(filePath)}
+
+${text}` };
+  }
+  if (mimeType.includes("wordprocessingml")) {
+    const text = await extractDocx(filePath);
+    return { title: basename(filePath), markdown: `# ${basename(filePath)}
+
+${text}` };
+  }
+  throw new Error(`unsupported file type: ${mimeType}`);
+}
+async function extractRawContent(raw, mimeType, fallbackTitle) {
+  if (mimeType === "text/markdown") {
+    const title = raw.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? fallbackTitle;
+    return { title, markdown: raw };
+  }
+  if (mimeType === "text/plain") {
+    return { title: fallbackTitle, markdown: `# ${fallbackTitle}
+
+${raw}` };
+  }
+  if (mimeType === "text/html") {
+    const extracted = extractHtmlToMarkdown(raw);
+    return { title: extracted.title, markdown: `# ${extracted.title}
+
+${extracted.markdown}` };
+  }
+  throw new Error(`raw reprocessing is not supported for ${mimeType}`);
+}
+async function ingestFile({
+  workspacePath,
+  source,
+  filePath,
+  previous
+}) {
+  const resolved = resolve(filePath);
+  const fileStat = await stat3(resolved);
+  const mimeType = mimeTypeFor(resolved);
+  const extracted = await extractFileContent(resolved, mimeType);
+  const documentId = stableId("doc", source.id, resolved);
+  const normalizedPath = resolve(workspacePath, "normalized", `${documentId}.md`);
+  const rawPath = resolve(workspacePath, "raw", source.id, basename(resolved));
+  const contentHash = sha256(extracted.markdown);
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
+  const indexedAt = now;
+  const crawledAt = now;
+  await mkdir7(resolve(workspacePath, "normalized"), { recursive: true });
+  await mkdir7(resolve(workspacePath, "raw", source.id), { recursive: true });
+  if (extracted.raw) {
+    await writeFile6(rawPath, extracted.raw, "utf8");
+  }
+  await writeNormalizedDocument({
+    documentId,
+    sourceId: source.id,
+    title: extracted.title,
+    uri: resolved,
+    sourceUri: source.uri,
+    publicationDate: previous?.publicationDate ?? null,
+    crawledAt,
+    indexedAt,
+    contentHash,
+    lastChangedAt,
+    normalizedPath,
+    markdown: extracted.markdown
+  });
+  return {
+    id: documentId,
+    sourceId: source.id,
+    sourceType: source.type,
+    title: extracted.title,
+    uri: resolved,
+    sourceUri: source.uri,
+    mimeType,
+    rawPath: extracted.raw ? rawPath : void 0,
+    normalizedPath,
+    contentHash,
+    metadata: buildDocumentMetadata({
+      source,
+      sourceUri: source.uri,
+      publicationDate: previous?.publicationDate ?? null,
+      crawledAt,
+      indexedAt,
+      extra: {
+        contentType: mimeType,
+        fileSizeBytes: fileStat.size
+      }
+    }),
+    publicationDate: previous?.publicationDate ?? null,
+    crawledAt,
+    firstSeenAt: previous?.firstSeenAt ?? now,
+    lastSeenAt: now,
+    lastChangedAt,
+    indexedAt
+  };
+}
+async function ingestInlineContent({
+  workspacePath,
+  source,
+  content,
+  title,
+  uri,
+  previous
+}) {
+  const markdown = source.type === "markdown" ? content : `# ${title}
+
+${content}`;
+  const documentId = stableId("doc", source.id, uri);
+  const normalizedPath = resolve(workspacePath, "normalized", `${documentId}.md`);
+  const contentHash = sha256(markdown);
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
+  const indexedAt = now;
+  await mkdir7(resolve(workspacePath, "normalized"), { recursive: true });
+  await writeNormalizedDocument({
+    documentId,
+    sourceId: source.id,
+    title,
+    uri,
+    sourceUri: source.uri,
+    publicationDate: previous?.publicationDate ?? null,
+    crawledAt: now,
+    indexedAt,
+    contentHash,
+    lastChangedAt,
+    normalizedPath,
+    markdown
+  });
+  return {
+    id: documentId,
+    sourceId: source.id,
+    sourceType: source.type,
+    title,
+    uri,
+    sourceUri: source.uri,
+    mimeType: source.type === "markdown" ? "text/markdown" : "text/plain",
+    normalizedPath,
+    contentHash,
+    metadata: buildDocumentMetadata({
+      source,
+      sourceUri: source.uri,
+      publicationDate: previous?.publicationDate ?? null,
+      crawledAt: now,
+      indexedAt
+    }),
+    publicationDate: previous?.publicationDate ?? null,
+    crawledAt: now,
+    firstSeenAt: previous?.firstSeenAt ?? now,
+    lastSeenAt: now,
+    lastChangedAt,
+    indexedAt
+  };
+}
+async function reprocessStoredDocument(document, source) {
+  if (!document.rawPath) {
+    return null;
+  }
+  const raw = await readFile9(document.rawPath, "utf8");
+  const fallbackTitle = document.title || basename(document.uri);
+  const extracted = await extractRawContent(raw, document.mimeType, fallbackTitle);
+  const contentHash = sha256(extracted.markdown);
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const indexedAt = now;
+  const lastChangedAt = document.contentHash === contentHash ? document.lastChangedAt : now;
+  await writeNormalizedDocument({
+    documentId: document.id,
+    sourceId: document.sourceId,
+    title: extracted.title,
+    uri: document.uri,
+    sourceUri: document.sourceUri,
+    publicationDate: document.publicationDate ?? null,
+    crawledAt: document.crawledAt,
+    indexedAt,
+    contentHash,
+    lastChangedAt,
+    normalizedPath: document.normalizedPath,
+    markdown: extracted.markdown
+  });
+  return {
+    ...document,
+    title: extracted.title,
+    contentHash,
+    metadata: buildDocumentMetadata({
+      source,
+      sourceUri: document.sourceUri,
+      publicationDate: document.publicationDate ?? null,
+      crawledAt: document.crawledAt,
+      indexedAt,
+      extra: {
+        ...document.metadata,
+        contentType: document.mimeType
+      }
+    }),
+    lastChangedAt,
+    indexedAt
+  };
+}
+
+// src/ingest/adapters/rss-adapter.ts
+import { Readable } from "stream";
+import FeedParser from "feedparser";
+import { parseFeed } from "feedsmith";
+function toIsoDate(value) {
+  if (value instanceof Date) {
+    return Number.isNaN(value.getTime()) ? null : value.toISOString();
+  }
+  if (typeof value === "string" && value.trim().length > 0) {
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString();
+  }
+  return null;
+}
+function normalizeFeedLink(link, baseUrl) {
+  if (!link?.trim()) {
+    return null;
+  }
+  try {
+    return new URL(link, baseUrl).href;
+  } catch {
+    return null;
+  }
+}
+function normalizeFeedsmithItems(feed, baseUrl) {
+  const items = Array.isArray(feed?.items) ? feed.items : Array.isArray(feed?.entries) ? feed.entries : [];
+  return items.map((item) => {
+    const link = normalizeFeedLink(
+      item?.link ?? item?.url ?? item?.id ?? item?.guid ?? item?.links?.[0]?.href ?? item?.links?.[0]?.href,
+      baseUrl
+    );
+    if (!link) {
+      return null;
+    }
+    return {
+      url: link,
+      title: String(item?.title ?? item?.summary ?? link).trim(),
+      publicationDate: toIsoDate(
+        item?.pubDate ?? item?.published ?? item?.updated ?? item?.published_at ?? item?.date_published ?? item?.dc?.date
+      )
+    };
+  }).filter((item) => item !== null);
+}
+async function parseWithFeedparser(xml, feedUrl) {
+  const parser = new FeedParser({ feedurl: feedUrl });
+  const items = [];
+  return await new Promise((resolve2, reject) => {
+    parser.on("error", reject);
+    parser.on("readable", function onReadable() {
+      let item;
+      while (item = this.read()) {
+        const link = normalizeFeedLink(item.link || item.origlink, feedUrl);
+        if (!link) {
+          continue;
+        }
+        items.push({
+          url: link,
+          title: String(item.title ?? link).trim(),
+          publicationDate: toIsoDate(item.pubdate ?? item.date)
+        });
+      }
+    });
+    parser.on("end", () => resolve2(items));
+    Readable.from([xml]).pipe(parser);
+  });
+}
+async function parseRssFeedDocument(xml, source) {
+  try {
+    const parsed = parseFeed(xml);
+    return normalizeFeedsmithItems(parsed.feed, source.uri);
+  } catch {
+    return parseWithFeedparser(xml, source.uri);
+  }
+}
+
+// src/ingest/adapters/url-adapter.ts
+import { mkdir as mkdir8, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
+import path16 from "path";
+function buildHttpCache(response2, validatedAt) {
+  return {
+    etag: response2.headers.get("etag") ?? void 0,
+    lastModified: response2.headers.get("last-modified") ?? void 0,
+    cacheControl: response2.headers.get("cache-control") ?? void 0,
+    expires: response2.headers.get("expires"),
+    lastValidatedAt: validatedAt,
+    lastStatus: response2.status
+  };
+}
+function choosePublicationDate(preferred, fallback, previous) {
+  return preferred ?? fallback ?? previous ?? null;
+}
+async function normalizeRemoteDocument({
+  workspacePath,
+  source,
+  url,
+  body,
+  previous,
+  sourceUri,
+  publicationDate,
+  responseStatus
+}) {
+  const extracted = extractHtmlToMarkdown(body);
+  const markdown = `# ${extracted.title}
+
+${extracted.markdown}`;
+  const documentId = stableId("doc", source.id, url);
+  const normalizedPath = path16.resolve(workspacePath, "normalized", `${documentId}.md`);
+  const rawPath = path16.resolve(workspacePath, "raw", source.id, `${sha256(url).slice(0, 12)}.html`);
+  const contentHash = sha256(markdown);
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
+  const indexedAt = now;
+  const crawledAt = now;
+  const resolvedPublicationDate = choosePublicationDate(publicationDate, extractPublicationDateFromHtml(body), previous?.publicationDate);
+  await mkdir8(path16.resolve(workspacePath, "raw", source.id), { recursive: true });
+  await writeFile7(rawPath, body, "utf8");
+  await writeNormalizedDocument({
+    documentId,
+    sourceId: source.id,
+    title: extracted.title,
+    uri: url,
+    sourceUri,
+    publicationDate: resolvedPublicationDate,
+    crawledAt,
+    indexedAt,
+    contentHash,
+    lastChangedAt,
+    normalizedPath,
+    markdown
+  });
+  return {
+    id: documentId,
+    sourceId: source.id,
+    sourceType: source.type,
+    title: extracted.title,
+    uri: url,
+    sourceUri,
+    mimeType: "text/html",
+    rawPath,
+    normalizedPath,
+    contentHash,
+    metadata: buildDocumentMetadata({
+      source,
+      sourceUri,
+      publicationDate: resolvedPublicationDate,
+      crawledAt,
+      indexedAt,
+      extra: {
+        status: responseStatus,
+        contentType: "text/html"
+      }
+    }),
+    publicationDate: resolvedPublicationDate,
+    crawledAt,
+    firstSeenAt: previous?.firstSeenAt ?? now,
+    lastSeenAt: now,
+    lastChangedAt,
+    indexedAt
+  };
+}
+async function fetchUrlDocument({
+  workspacePath,
+  source,
+  url,
+  previous,
+  sourceUri,
+  publicationDate
+}) {
+  const headers = {
+    "user-agent": source.crawl?.userAgent ?? "querylight-cli/0.1"
+  };
+  if (previous?.httpCache?.etag) {
+    headers["if-none-match"] = previous.httpCache.etag;
+  }
+  if (previous?.httpCache?.lastModified) {
+    headers["if-modified-since"] = previous.httpCache.lastModified;
+  }
+  const response2 = await fetch(url, { headers });
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const nextHttpCache = buildHttpCache(response2, now);
+  const effectiveSourceUri = sourceUri ?? source.uri;
+  if (response2.status === 304 && previous?.rawPath && await fileExists(previous.rawPath) && await fileExists(previous.normalizedPath)) {
+    return {
+      ...previous,
+      sourceUri: effectiveSourceUri,
+      publicationDate: publicationDate ?? previous.publicationDate ?? null,
+      metadata: buildDocumentMetadata({
+        source,
+        sourceUri: effectiveSourceUri,
+        publicationDate: publicationDate ?? previous.publicationDate ?? null,
+        crawledAt: previous.crawledAt,
+        indexedAt: previous.indexedAt,
+        extra: {
+          ...previous.metadata,
+          status: previous.metadata.status ?? 200,
+          contentType: previous.mimeType
+        }
+      }),
+      lastSeenAt: now,
+      httpCache: nextHttpCache
+    };
+  }
+  const body = await response2.text();
+  const document = await normalizeRemoteDocument({
+    workspacePath,
+    source,
+    url,
+    body,
+    previous,
+    sourceUri: effectiveSourceUri,
+    publicationDate,
+    responseStatus: response2.status
+  });
+  return {
+    ...document,
+    mimeType: response2.headers.get("content-type") ?? document.mimeType,
+    metadata: buildDocumentMetadata({
+      source,
+      sourceUri: effectiveSourceUri,
+      publicationDate: document.publicationDate ?? null,
+      crawledAt: document.crawledAt,
+      indexedAt: document.indexedAt,
+      extra: {
+        status: response2.status,
+        contentType: response2.headers.get("content-type") ?? document.mimeType
+      }
+    }),
+    httpCache: nextHttpCache
+  };
+}
+async function reprocessRemoteDocument(document, source) {
+  if (!document.rawPath || !await fileExists(document.rawPath)) {
+    return null;
+  }
+  const raw = await readFile10(document.rawPath, "utf8");
+  const extracted = extractHtmlToMarkdown(raw);
+  const markdown = `# ${extracted.title}
+
+${extracted.markdown}`;
+  const contentHash = sha256(markdown);
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const indexedAt = now;
+  const lastChangedAt = document.contentHash === contentHash ? document.lastChangedAt : now;
+  const publicationDate = document.publicationDate ?? extractPublicationDateFromHtml(raw);
+  await writeNormalizedDocument({
+    documentId: document.id,
+    sourceId: document.sourceId,
+    title: extracted.title,
+    uri: document.uri,
+    sourceUri: document.sourceUri,
+    publicationDate,
+    crawledAt: document.crawledAt,
+    indexedAt,
+    contentHash,
+    lastChangedAt,
+    normalizedPath: document.normalizedPath,
+    markdown
+  });
+  return {
+    ...document,
+    title: extracted.title,
+    contentHash,
+    publicationDate,
+    metadata: buildDocumentMetadata({
+      source,
+      sourceUri: document.sourceUri,
+      publicationDate,
+      crawledAt: document.crawledAt,
+      indexedAt,
+      extra: {
+        ...document.metadata,
+        status: document.httpCache?.lastStatus ?? document.metadata.status ?? 200,
+        contentType: document.mimeType
+      }
+    }),
+    lastChangedAt,
+    indexedAt
+  };
+}
+
+// src/ingest/adapters/website-adapter.ts
+import { load as load2 } from "cheerio";
+async function fetchRobotsDisallow(url, userAgent) {
+  try {
+    const response2 = await fetch(new URL("/robots.txt", url), { headers: { "user-agent": userAgent } });
+    if (!response2.ok) {
+      return [];
+    }
+    const text = await response2.text();
+    return text.split("\n").map((line) => line.trim()).filter((line) => /^disallow:/i.test(line)).map((line) => line.split(":")[1]?.trim() ?? "").filter((line) => line.length > 0);
+  } catch {
+    return [];
+  }
+}
+async function fetchSitemapUrls(baseUrl, userAgent) {
+  try {
+    const response2 = await fetch(new URL("/sitemap.xml", baseUrl), { headers: { "user-agent": userAgent } });
+    if (!response2.ok) {
+      return [];
+    }
+    const xml = await response2.text();
+    return [...xml.matchAll(/<loc>(.*?)<\/loc>/g)].map((match) => match[1]).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+function isAllowed(url, baseUrl, includePatterns, excludePatterns, disallowRules) {
+  if (url.origin !== baseUrl.origin) {
+    return false;
+  }
+  if (disallowRules.some((rule) => rule !== "/" && url.pathname.startsWith(rule))) {
+    return false;
+  }
+  const href = url.href;
+  if (includePatterns.length > 0 && !includePatterns.some((pattern) => href.includes(pattern))) {
+    return false;
+  }
+  if (excludePatterns.some((pattern) => href.includes(pattern))) {
+    return false;
+  }
+  return true;
+}
+async function crawlWebsite(source) {
+  const baseUrl = new URL(source.uri);
+  const userAgent = source.crawl?.userAgent ?? "querylight-cli/0.1";
+  const includePatterns = source.crawl?.includePatterns ?? [];
+  const excludePatterns = source.crawl?.excludePatterns ?? [];
+  const maxDepth = source.crawl?.maxDepth ?? 2;
+  const maxPages = source.crawl?.maxPages ?? 100;
+  const rateLimitMs = source.crawl?.rateLimitMs ?? 1e3;
+  const disallowRules = source.crawl?.obeyRobotsTxt === false ? [] : await fetchRobotsDisallow(baseUrl, userAgent);
+  const queue = [{ url: source.uri, depth: 0 }];
+  const seen = /* @__PURE__ */ new Set();
+  const results = [];
+  if (source.crawl?.useSitemap !== false) {
+    for (const url of await fetchSitemapUrls(baseUrl, userAgent)) {
+      queue.push({ url, depth: 1 });
+    }
+  }
+  while (queue.length > 0 && results.length < maxPages) {
+    const next = queue.shift();
+    if (!next || seen.has(next.url)) {
+      continue;
+    }
+    seen.add(next.url);
+    const url = new URL(next.url);
+    if (!isAllowed(url, baseUrl, includePatterns, excludePatterns, disallowRules)) {
+      continue;
+    }
+    results.push(url.href);
+    if (next.depth >= maxDepth) {
+      continue;
+    }
+    const response2 = await fetch(url, { headers: { "user-agent": userAgent } });
+    const html = await response2.text();
+    const $ = load2(html);
+    $("a[href]").each((_, element) => {
+      const href = $(element).attr("href");
+      if (!href) {
+        return;
+      }
+      try {
+        const target = new URL(href, url);
+        if (!seen.has(target.href)) {
+          queue.push({ url: target.href, depth: next.depth + 1 });
+        }
+      } catch {
+      }
+    });
+    if (rateLimitMs > 0) {
+      await new Promise((resolve2) => setTimeout(resolve2, rateLimitMs));
+    }
+  }
+  return results;
+}
+
+// src/ingest/ingest-service.ts
+function documentsFile(workspacePath) {
+  return path17.join(workspacePath, "documents", "documents.jsonl");
+}
+async function loadDocuments(workspacePath) {
+  return readJsonl(documentsFile(workspacePath));
+}
+async function saveDocuments(workspacePath, documents) {
+  await writeJsonl(documentsFile(workspacePath), documents.sort((a, b) => a.id.localeCompare(b.id)));
+}
+function previousMap(documents) {
+  return new Map(documents.map((document) => [document.id, document]));
+}
+function nowStamp() {
+  return (/* @__PURE__ */ new Date()).toISOString();
+}
+function runId() {
+  return nowStamp().replace(/[:.]/g, "-");
+}
+function documentSnapshot(documents) {
+  return documents.map((document) => ({
+    id: document.id,
+    title: document.title,
+    uri: document.uri,
+    contentHash: document.contentHash,
+    lastChangedAt: document.lastChangedAt,
+    sourceId: document.sourceId
+  }));
+}
+function shouldExpireRssDocument(document, source, defaultRetentionDays) {
+  if (source.type !== "rss" || !document.publicationDate) {
+    return false;
+  }
+  const retentionDays = source.crawl?.retentionDays ?? defaultRetentionDays;
+  const publishedAt = new Date(document.publicationDate);
+  if (Number.isNaN(publishedAt.getTime())) {
+    return false;
+  }
+  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1e3;
+  return publishedAt.getTime() < cutoff;
+}
+async function purgeDocuments(workspacePath, documentIds, documents) {
+  if (documentIds.size === 0) {
+    return;
+  }
+  const chunks = await loadChunks(workspacePath);
+  const filteredChunks = chunks.filter((chunk) => !documentIds.has(chunk.documentId));
+  if (filteredChunks.length !== chunks.length) {
+    await saveChunks(workspacePath, filteredChunks);
+  }
+  await Promise.all(
+    documents.filter((document) => documentIds.has(document.id)).map((document) => deleteDocumentArtifacts(document))
+  );
+}
+async function fetchFeedText(source) {
+  const response2 = await fetch(source.uri, {
+    headers: {
+      "user-agent": source.crawl?.userAgent ?? "querylight-cli/0.1"
+    }
+  });
+  if (!response2.ok) {
+    throw new Error(`failed to fetch feed: ${response2.status}`);
+  }
+  return response2.text();
+}
+async function ingestRssSource({
+  workspacePath,
+  source,
+  previous,
+  nextDocuments,
+  onFailure
+}) {
+  if (source.crawl?.fetchArticles === false) {
+    throw new Error("rss sources require article fetching");
+  }
+  const xml = await fetchFeedText(source);
+  const items = await parseRssFeedDocument(xml, source);
+  let added = 0;
+  let changed = 0;
+  let unchanged = 0;
+  let failed = 0;
+  for (const item of items) {
+    try {
+      const probe = previous.get(stableId("doc", source.id, item.url));
+      const document = await fetchUrlDocument({
+        workspacePath,
+        source,
+        url: item.url,
+        previous: probe,
+        sourceUri: source.uri,
+        publicationDate: item.publicationDate
+      });
+      nextDocuments.set(document.id, document);
+      if (!probe) {
+        added += 1;
+      } else if (probe.contentHash !== document.contentHash) {
+        changed += 1;
+      } else {
+        unchanged += 1;
+      }
+    } catch (error) {
+      failed += 1;
+      onFailure(item.url, error);
+    }
+  }
+  return { added, changed, unchanged, failed };
+}
+async function ingestSources({
+  workspacePath,
+  sourceIds,
+  changedOnly = false
+}) {
+  const config = await loadConfig(workspacePath);
+  const defaultRetentionDays = config.crawler.retentionDays;
+  const sources = (await listSources(workspacePath)).filter((source) => source.enabled && (!sourceIds || sourceIds.includes(source.id)));
+  const existing = await loadDocuments(workspacePath);
+  const previous = previousMap(existing);
+  const nextDocuments = new Map(existing.map((document) => [document.id, document]));
+  let added = 0;
+  let changed = 0;
+  let unchanged = 0;
+  let failed = 0;
+  const failures = [];
+  for (const source of sources) {
+    const ingestOne = async (uri, producer) => {
+      try {
+        const probeId = stableId("doc", source.id, uri);
+        const earlier = previous.get(probeId);
+        const document = await producer();
+        nextDocuments.set(document.id, document);
+        if (!earlier) {
+          added += 1;
+        } else if (earlier.contentHash !== document.contentHash) {
+          changed += 1;
+        } else {
+          unchanged += 1;
+        }
+      } catch (error) {
+        failed += 1;
+        failures.push({
+          sourceId: source.id,
+          uri,
+          message: error instanceof Error ? error.message : String(error)
+        });
+      }
+    };
+    try {
+      if (source.type === "file") {
+        await ingestOne(source.uri, () => ingestFile({ workspacePath, source, filePath: source.uri, previous: previous.get(stableId("doc", source.id, source.uri)) }));
+        continue;
+      }
+      if (source.type === "directory") {
+        for (const filePath of await listDirectoryFiles(source)) {
+          await ingestOne(filePath, () => ingestFile({ workspacePath, source, filePath, previous: previous.get(stableId("doc", source.id, filePath)) }));
+        }
+        continue;
+      }
+      if (source.type === "url") {
+        await ingestOne(source.uri, () => fetchUrlDocument({ workspacePath, source, url: source.uri, previous: previous.get(stableId("doc", source.id, source.uri)) }));
+        continue;
+      }
+      if (source.type === "website") {
+        for (const url of await crawlWebsite(source)) {
+          await ingestOne(url, () => fetchUrlDocument({ workspacePath, source, url, previous: previous.get(stableId("doc", source.id, url)) }));
+        }
+        continue;
+      }
+      if (source.type === "rss") {
+        const result2 = await ingestRssSource({
+          workspacePath,
+          source,
+          previous,
+          nextDocuments,
+          onFailure: (uri, error) => {
+            failures.push({
+              sourceId: source.id,
+              uri,
+              message: error instanceof Error ? error.message : String(error)
+            });
+          }
+        });
+        added += result2.added;
+        changed += result2.changed;
+        unchanged += result2.unchanged;
+        failed += result2.failed;
+        continue;
+      }
+      if (source.type === "markdown" || source.type === "text") {
+        await ingestOne(source.uri, () => ingestInlineContent({
+          workspacePath,
+          source,
+          title: source.name,
+          content: source.uri,
+          uri: `inline:${source.id}`,
+          previous: previous.get(stableId("doc", source.id, `inline:${source.id}`))
+        }));
+      }
+    } catch (error) {
+      failed += 1;
+      failures.push({
+        sourceId: source.id,
+        uri: source.uri,
+        message: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+  const expiringDocuments = [...nextDocuments.values()].filter((document) => {
+    const source = sources.find((candidate) => candidate.id === document.sourceId);
+    return source ? shouldExpireRssDocument(document, source, defaultRetentionDays) : false;
+  });
+  if (expiringDocuments.length > 0) {
+    const expiredIds = new Set(expiringDocuments.map((document) => document.id));
+    for (const document of expiringDocuments) {
+      nextDocuments.delete(document.id);
+    }
+    await purgeDocuments(workspacePath, expiredIds, [...existing, ...expiringDocuments]);
+  }
+  const finalDocuments = [...nextDocuments.values()];
+  await saveDocuments(workspacePath, finalDocuments);
+  const id = runId();
+  const run = {
+    id,
+    kind: "ingest",
+    createdAt: nowStamp(),
+    success: failed === 0,
+    summary: {
+      processedSources: sources.length,
+      added,
+      changed,
+      unchanged,
+      failed,
+      changedOnly
+    },
+    failures,
+    documentsSnapshot: documentSnapshot(finalDocuments)
+  };
+  await writeRun(workspacePath, run);
+  return {
+    runId: id,
+    documents: { added, changed, unchanged, failed },
+    processedSources: sources.length
+  };
+}
+async function reprocessDocuments({
+  workspacePath,
+  sourceId,
+  documentId
+}) {
+  const documents = await loadDocuments(workspacePath);
+  const sources = await listSources(workspacePath);
+  const sourceMap = new Map(sources.map((source) => [source.id, source]));
+  const nextDocuments = new Map(documents.map((document) => [document.id, document]));
+  let documentsReprocessed = 0;
+  let documentsSkipped = 0;
+  for (const document of documents.filter((candidate) => (!sourceId || candidate.sourceId === sourceId) && (!documentId || candidate.id === documentId))) {
+    const source = sourceMap.get(document.sourceId);
+    if (!source || !document.rawPath || !await fileExists(document.rawPath)) {
+      documentsSkipped += 1;
+      continue;
+    }
+    const updated = source.type === "url" || source.type === "website" || source.type === "rss" ? await reprocessRemoteDocument(document, source) : await reprocessStoredDocument(document, source);
+    if (!updated) {
+      documentsSkipped += 1;
+      continue;
+    }
+    nextDocuments.set(updated.id, updated);
+    documentsReprocessed += 1;
+  }
+  const finalDocuments = [...nextDocuments.values()];
+  await saveDocuments(workspacePath, finalDocuments);
+  const id = runId();
+  await writeRun(workspacePath, {
+    id,
+    kind: "reprocess",
+    createdAt: nowStamp(),
+    success: true,
+    summary: {
+      documentsReprocessed,
+      documentsSkipped
+    },
+    documentsSnapshot: documentSnapshot(finalDocuments)
+  });
+  return { runId: id, documentsReprocessed, documentsSkipped };
+}
+
+// src/query/search-service.ts
+import { readFile as readFile11 } from "fs/promises";
+import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
+import path18 from "path";
+async function loadHydratedIndex(workspacePath) {
+  const state = await readLatestIndexState(workspacePath);
+  const mapping = createIndexMapping(Object.keys(state.fieldState ?? {}).filter((field) => field.startsWith("metadata.")));
+  return new (await import("@tryformation/querylight-ts")).DocumentIndex(mapping).loadState(state);
+}
+function normalizeFilterValues(values) {
+  return (values ?? []).map((value) => value.toLowerCase()).filter(Boolean);
+}
+function matchesAny(value, candidates) {
+  return candidates.length === 0 || candidates.includes(value.toLowerCase());
+}
+function matchesPrefix(value, prefixes) {
+  if (prefixes.length === 0) {
+    return true;
+  }
+  const lower = value.toLowerCase();
+  return prefixes.some((prefix) => lower.startsWith(prefix));
+}
+function buildSearchQuery(query, filters) {
+  const sourceIds = normalizeFilterValues([filters.sourceId, ...filters.sourceIds ?? []].filter((value) => Boolean(value)));
+  const sourceTypes = normalizeFilterValues([filters.sourceType, ...filters.sourceTypes ?? []].filter((value) => Boolean(value)));
+  const tags = normalizeFilterValues([filters.tag, ...filters.tags ?? []].filter((value) => Boolean(value)));
+  return new BoolQuery({
+    should: [
+      new MatchQuery({ field: "title", text: query, operation: OP.AND, boost: 6 }),
+      new MatchQuery({ field: "text", text: query, operation: OP.AND, boost: 4 }),
+      new MatchQuery({ field: "text", text: query, operation: OP.OR, boost: 2 })
+    ],
+    filter: [
+      ...sourceIds.length === 1 ? [new TermQuery({ field: "sourceId", text: sourceIds[0] })] : [],
+      ...sourceTypes.length === 1 ? [new TermQuery({ field: "sourceType", text: sourceTypes[0] })] : [],
+      ...tags.length === 1 ? [new TermQuery({ field: "tags", text: tags[0] })] : [],
+      ...(filters.metadata ?? []).map(({ key, value }) => new TermQuery({ field: `metadata.${key}`, text: value.toLowerCase() }))
+    ]
+  });
+}
+function isValidDate(value) {
+  return typeof value === "string" && !Number.isNaN(new Date(value).getTime());
+}
+function documentDateValue(document, field) {
+  const value = document[field];
+  return typeof value === "string" && isValidDate(value) ? value : null;
+}
+function matchesDateRanges(document, dateRanges) {
+  return dateRanges.every(({ field, from, to }) => {
+    const value = documentDateValue(document, field);
+    if (!value) {
+      return false;
+    }
+    const timestamp = new Date(value).getTime();
+    const fromTime = from ? new Date(from).getTime() : null;
+    const toTime = to ? new Date(to).getTime() : null;
+    return (fromTime == null || timestamp >= fromTime) && (toTime == null || timestamp <= toTime);
+  });
+}
+function fallbackSourceType(chunk, document, source) {
+  const metadataSourceType = typeof chunk.metadata.sourceType === "string" ? chunk.metadata.sourceType : void 0;
+  return document?.sourceType ?? source?.type ?? metadataSourceType ?? "text";
+}
+function filterChunk(chunk, document, source, {
+  sourceId,
+  sourceIds,
+  sourceName,
+  sourceNames,
+  sourceType,
+  sourceTypes,
+  uriPrefix,
+  uriPrefixes,
+  hasPublicationDate,
+  tag,
+  tags,
+  metadata,
+  dateRanges
+}) {
+  const normalizedSourceIds = normalizeFilterValues([sourceId, ...sourceIds ?? []].filter((value) => Boolean(value)));
+  const normalizedSourceNames = normalizeFilterValues([sourceName, ...sourceNames ?? []].filter((value) => Boolean(value)));
+  const normalizedSourceTypes = normalizeFilterValues([sourceType, ...sourceTypes ?? []].filter((value) => Boolean(value)));
+  const normalizedUriPrefixes = normalizeFilterValues([uriPrefix, ...uriPrefixes ?? []].filter((value) => Boolean(value)));
+  const normalizedTags = normalizeFilterValues([tag, ...tags ?? []].filter((value) => Boolean(value)));
+  if (!matchesAny(chunk.sourceId, normalizedSourceIds)) {
+    return false;
+  }
+  if (!matchesAny(fallbackSourceType(chunk, document, source), normalizedSourceTypes)) {
+    return false;
+  }
+  if (normalizedSourceNames.length > 0 && !matchesAny(source?.name ?? "", normalizedSourceNames)) {
+    return false;
+  }
+  if (!matchesPrefix(document?.uri ?? chunk.uri, normalizedUriPrefixes)) {
+    return false;
+  }
+  if (hasPublicationDate && (!document || !documentDateValue(document, "publicationDate"))) {
+    return false;
+  }
+  if (normalizedTags.length > 0) {
+    const tags2 = Array.isArray(chunk.metadata.tags) ? chunk.metadata.tags.map(String).map((value) => value.toLowerCase()) : [];
+    if (!normalizedTags.some((tag2) => tags2.includes(tag2))) {
+      return false;
+    }
+  }
+  if (metadata?.length) {
+    const metadataMatches = metadata.every(({ key, value }) => {
+      const candidate = chunk.metadata[key];
+      return Array.isArray(candidate) ? candidate.map(String).map((item) => item.toLowerCase()).includes(value.toLowerCase()) : String(candidate ?? "").toLowerCase() === value.toLowerCase();
+    });
+    if (!metadataMatches) {
+      return false;
+    }
+  }
+  if (!document) {
+    return dateRanges.length === 0;
+  }
+  return matchesDateRanges(document, dateRanges);
+}
+function sortDateDescending(left, right) {
+  const leftTime = left ? new Date(left).getTime() : Number.NEGATIVE_INFINITY;
+  const rightTime = right ? new Date(right).getTime() : Number.NEGATIVE_INFINITY;
+  return rightTime - leftTime;
+}
+function latestSortDate(document) {
+  return documentDateValue(document, "publicationDate") ?? documentDateValue(document, "lastChangedAt") ?? documentDateValue(document, "lastSeenAt") ?? documentDateValue(document, "firstSeenAt") ?? documentDateValue(document, "crawledAt");
+}
+function representativeChunk(chunks) {
+  return [...chunks].sort((left, right) => {
+    if (left.headingPath.length !== right.headingPath.length) {
+      return left.headingPath.length - right.headingPath.length;
+    }
+    if (left.uri !== right.uri) {
+      return left.uri.localeCompare(right.uri);
+    }
+    return left.id.localeCompare(right.id);
+  })[0] ?? chunks[0] ?? void 0;
+}
+function stripSnippetMarkdown(text) {
+  return text.replace(/```[\s\S]*?```/g, " ").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/`([^`]+)`/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/^\s*[-*+]\s+/gm, "");
+}
+function extractSnippetParagraphs(text) {
+  return stripSnippetMarkdown(text).split(/\n\s*\n+/).map((paragraph) => paragraph.replace(/\s+/g, " ").trim()).filter(Boolean);
+}
+function buildParagraphSnippet(paragraphs, query, targetLength = 900) {
+  if (paragraphs.length === 0) {
+    return "";
+  }
+  const lowerQueryTerms = query.toLowerCase().split(/\s+/).filter(Boolean);
+  const matchIndex = paragraphs.findIndex((paragraph) => {
+    const lower = paragraph.toLowerCase();
+    return lowerQueryTerms.some((term) => lower.includes(term));
+  });
+  let start = matchIndex >= 0 ? matchIndex : 0;
+  let end = start + 1;
+  let totalLength = paragraphs[start]?.length ?? 0;
+  while (totalLength < targetLength && (start > 0 || end < paragraphs.length)) {
+    const previousLength = start > 0 ? paragraphs[start - 1]?.length ?? 0 : -1;
+    const nextLength = end < paragraphs.length ? paragraphs[end]?.length ?? 0 : -1;
+    if (nextLength >= previousLength && end < paragraphs.length) {
+      totalLength += nextLength + 2;
+      end += 1;
+      continue;
+    }
+    if (start > 0) {
+      totalLength += previousLength + 2;
+      start -= 1;
+      continue;
+    }
+    break;
+  }
+  return paragraphs.slice(start, end).join("\n\n").trim();
+}
+function buildSnippet(text, query) {
+  return buildParagraphSnippet(extractSnippetParagraphs(text), query);
+}
+function buildDocumentParagraphs(chunks) {
+  return chunks.flatMap(
+    (candidate, chunkIndex) => extractSnippetParagraphs(candidate.text).map((text) => ({ chunkIndex, text }))
+  );
+}
+function buildExpandedParagraphSnippet(paragraphs, chunkIndex, query, targetLength = 1200) {
+  if (paragraphs.length === 0) {
+    return "";
+  }
+  const lowerQueryTerms = query.toLowerCase().split(/\s+/).filter(Boolean);
+  const currentParagraphIndexes = paragraphs.map((paragraph, index) => ({ ...paragraph, index })).filter((paragraph) => paragraph.chunkIndex === chunkIndex).map((paragraph) => paragraph.index);
+  const anchorIndex = currentParagraphIndexes.find((index) => {
+    const lower = paragraphs[index]?.text.toLowerCase() ?? "";
+    return lowerQueryTerms.some((term) => lower.includes(term));
+  }) ?? currentParagraphIndexes[0] ?? 0;
+  let start = anchorIndex;
+  let end = anchorIndex + 1;
+  let totalLength = paragraphs[anchorIndex]?.text.length ?? 0;
+  while (totalLength < targetLength && (start > 0 || end < paragraphs.length)) {
+    const previousLength = start > 0 ? paragraphs[start - 1]?.text.length ?? 0 : -1;
+    const nextLength = end < paragraphs.length ? paragraphs[end]?.text.length ?? 0 : -1;
+    if (nextLength >= previousLength && end < paragraphs.length) {
+      totalLength += nextLength + 2;
+      end += 1;
+      continue;
+    }
+    if (start > 0) {
+      totalLength += previousLength + 2;
+      start -= 1;
+      continue;
+    }
+    break;
+  }
+  return paragraphs.slice(start, end).map((paragraph) => paragraph.text).join("\n\n").trim();
+}
+async function buildSnippetWithAdjacentChunks(chunk, query, {
+  document,
+  config,
+  orderedChunkCache
+}) {
+  if (!document) {
+    return buildSnippet(chunk.text, query);
+  }
+  let orderedChunks = orderedChunkCache.get(document.id);
+  if (!orderedChunks) {
+    if (!await fileExists(document.normalizedPath)) {
+      return buildSnippet(chunk.text, query);
+    }
+    const raw = await readFile11(document.normalizedPath, "utf8");
+    orderedChunks = buildChunksForDocument(document, raw, config);
+    orderedChunkCache.set(document.id, orderedChunks);
+  }
+  const currentIndex = orderedChunks.findIndex((candidate) => candidate.id === chunk.id);
+  if (currentIndex < 0) {
+    return buildSnippet(chunk.text, query);
+  }
+  const current = orderedChunks[currentIndex];
+  const paragraphs = buildDocumentParagraphs(orderedChunks);
+  if (paragraphs.length === 0) {
+    return buildSnippet(current.text, query);
+  }
+  return buildExpandedParagraphSnippet(paragraphs, currentIndex, query);
+}
+function normalizeDisplayTitle(title) {
+  return title.replace(/\s*\|\s*Querylight TS Demo\s*$/i, "").replace(/\s+/g, " ").trim();
+}
+function chooseResultTitle(chunk) {
+  const documentTitle = normalizeDisplayTitle(chunk.title);
+  const headings = chunk.headingPath.map((heading) => normalizeDisplayTitle(heading)).filter(Boolean);
+  const leafHeading = headings.at(-1);
+  if (leafHeading && leafHeading.toLowerCase() !== documentTitle.toLowerCase()) {
+    return leafHeading;
+  }
+  if (documentTitle) {
+    return documentTitle;
+  }
+  return leafHeading ?? "Untitled";
+}
+function normalizeComparisonText(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").replace(/\s+/g, " ").trim();
+}
+function normalizeUriPath(uri) {
+  try {
+    const parsed = new URL(uri);
+    const pathname = parsed.pathname.replace(/\/+$/, "") || "/";
+    return pathname.toLowerCase();
+  } catch {
+    return uri.toLowerCase().replace(/\/+$/, "");
+  }
+}
+function uriSpecificity(uri) {
+  const normalized = normalizeUriPath(uri);
+  if (normalized === "/") {
+    return 0;
+  }
+  return normalized.split("/").filter(Boolean).length;
+}
+function isMoreSpecificDuplicate(candidate, existing) {
+  if (candidate.sourceId !== existing.sourceId) {
+    return false;
+  }
+  const candidateTitle = normalizeComparisonText(candidate.title);
+  const existingTitle = normalizeComparisonText(existing.title);
+  if (!candidateTitle || candidateTitle !== existingTitle) {
+    return false;
+  }
+  const candidatePath = normalizeUriPath(candidate.uri);
+  const existingPath = normalizeUriPath(existing.uri);
+  if (candidatePath === existingPath) {
+    return false;
+  }
+  const candidateIsChild = candidatePath.startsWith(existingPath === "/" ? "/" : `${existingPath}/`);
+  const existingIsChild = existingPath.startsWith(candidatePath === "/" ? "/" : `${candidatePath}/`);
+  if (!candidateIsChild && !existingIsChild) {
+    return false;
+  }
+  return uriSpecificity(candidate.uri) > uriSpecificity(existing.uri);
+}
+function collapseAggregateDuplicates(results, topK) {
+  const deduped = [];
+  for (const result2 of results) {
+    const duplicateIndex = deduped.findIndex(
+      (existing) => isMoreSpecificDuplicate(result2, existing) || isMoreSpecificDuplicate(existing, result2)
+    );
+    if (duplicateIndex < 0) {
+      deduped.push(result2);
+      continue;
+    }
+    if (isMoreSpecificDuplicate(result2, deduped[duplicateIndex])) {
+      deduped[duplicateIndex] = result2;
+    }
+  }
+  return deduped.slice(0, topK);
+}
+function rerankResultsByDocument(results, topK) {
+  const byDocument = /* @__PURE__ */ new Map();
+  for (const result2 of results) {
+    const existing = byDocument.get(result2.documentId);
+    if (existing) {
+      existing.push(result2);
+    } else {
+      byDocument.set(result2.documentId, [result2]);
+    }
+  }
+  const reranked = [...byDocument.values()].flatMap((group) => {
+    const sorted = [...group].sort((left, right) => right.score - left.score);
+    const [best, ...rest] = sorted;
+    if (!best) {
+      return [];
+    }
+    const tailScore = rest.reduce((sum, result2) => sum + result2.score, 0);
+    const aggregateScore = best.score + tailScore * 0.35 + (group.length - 1) * 0.2;
+    return [{ ...best, score: aggregateScore }];
+  }).sort((left, right) => right.score - left.score);
+  return collapseAggregateDuplicates(reranked, topK);
+}
+async function searchIndex({
+  workspacePath,
+  query,
+  topK,
+  sourceId,
+  sourceIds,
+  sourceName,
+  sourceNames,
+  sourceType,
+  sourceTypes,
+  uriPrefix,
+  uriPrefixes,
+  hasPublicationDate,
+  tag,
+  tags,
+  metadata,
+  dateRanges = [],
+  retrievalMode,
+  showChunks = false
+}) {
+  const config = await loadConfig(workspacePath);
+  const mode = retrievalMode ?? config.retrieval.defaultMode;
+  const candidateLimit = Math.max(topK * 5, 50);
+  const chunks = new Map((await readJsonl(path18.join(workspacePath, "chunks", "chunks.jsonl"))).map((chunk) => [chunk.id, chunk]));
+  const documents = new Map((await readJsonl(path18.join(workspacePath, "documents", "documents.jsonl"))).map((document) => [document.id, document]));
+  const sources = new Map((await readJsonl(path18.join(workspacePath, "sources", "sources.jsonl"))).map((source) => [source.id, source]));
+  const orderedChunkCache = /* @__PURE__ */ new Map();
+  const normalizedQuery = query.trim();
+  const filterIds = [...chunks.values()].filter((chunk) => filterChunk(chunk, documents.get(chunk.documentId), sources.get(chunk.sourceId), { sourceId, sourceIds, sourceName, sourceNames, sourceType, sourceTypes, uriPrefix, uriPrefixes, hasPublicationDate, tag, tags, metadata, dateRanges })).map((chunk) => chunk.id);
+  if (normalizedQuery.length === 0) {
+    const chunksByDocument = /* @__PURE__ */ new Map();
+    for (const chunkId of filterIds) {
+      const chunk = chunks.get(chunkId);
+      if (!chunk) {
+        continue;
+      }
+      const existing = chunksByDocument.get(chunk.documentId);
+      if (existing) {
+        existing.push(chunk);
+      } else {
+        chunksByDocument.set(chunk.documentId, [chunk]);
+      }
+    }
+    const latestResults = await Promise.all(
+      [...chunksByDocument.entries()].sort(([leftDocumentId], [rightDocumentId]) => {
+        const leftDocument = documents.get(leftDocumentId);
+        const rightDocument = documents.get(rightDocumentId);
+        return sortDateDescending(leftDocument ? latestSortDate(leftDocument) : null, rightDocument ? latestSortDate(rightDocument) : null);
+      }).slice(0, topK).map(async ([documentId, documentChunks]) => {
+        const document = documents.get(documentId);
+        const chunk = representativeChunk(documentChunks);
+        if (!chunk || !document) {
+          return null;
+        }
+        return {
+          chunkId: chunk.id,
+          documentId: chunk.documentId,
+          sourceId: chunk.sourceId,
+          sourceType: document.sourceType,
+          score: 0,
+          title: chooseResultTitle(chunk),
+          uri: chunk.uri,
+          headingPath: chunk.headingPath,
+          snippet: await buildSnippetWithAdjacentChunks(chunk, document.title, {
+            document,
+            config,
+            orderedChunkCache
+          }),
+          text: showChunks ? chunk.text : void 0,
+          publicationDate: document.publicationDate ?? null,
+          firstSeenAt: document.firstSeenAt,
+          lastSeenAt: document.lastSeenAt,
+          lastChangedAt: document.lastChangedAt,
+          metadata: chunk.metadata
+        };
+      })
+    );
+    return { retrievalMode: "lexical", results: latestResults.filter((result2) => result2 != null) };
+  }
+  const lexicalHits = async () => {
+    const index = await loadHydratedIndex(workspacePath);
+    const all = await index.searchRequest({ query: buildSearchQuery(normalizedQuery, { sourceId, sourceIds, sourceType, sourceTypes, tag, tags, metadata }), limit: candidateLimit });
+    return all.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit);
+  };
+  const denseHits = async () => {
+    if (!await fileExists(denseVectorPath(workspacePath))) {
+      throw new CliError("dense vector index is not built; run `qli models pull --dense` and `qli rebuild`", "DENSE_INDEX_MISSING", 7 /* QueryError */);
+    }
+    return denseQuery({ workspacePath, config: config.retrieval.dense, query: normalizedQuery, topK: candidateLimit }).then((hits2) => hits2.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit));
+  };
+  const sparseHits = async () => {
+    if (!await fileExists(sparseVectorPath(workspacePath))) {
+      throw new CliError("sparse vector index is not built; run `qli models pull --sparse` and `qli rebuild`", "SPARSE_INDEX_MISSING", 7 /* QueryError */);
+    }
+    return sparseQuery({ workspacePath, config: config.retrieval.sparse, query: normalizedQuery, topK: candidateLimit }).then((hits2) => hits2.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit));
+  };
+  let hits;
+  if (mode === "lexical") {
+    hits = await lexicalHits();
+  } else if (mode === "dense") {
+    hits = await denseHits();
+  } else if (mode === "sparse") {
+    hits = await sparseHits();
+  } else {
+    const rankings = [await lexicalHits()];
+    if (await fileExists(denseVectorPath(workspacePath))) {
+      rankings.push(await denseQuery({ workspacePath, config: config.retrieval.dense, query: normalizedQuery, topK: candidateLimit }).then((dense) => dense.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit)));
+    }
+    if (await fileExists(sparseVectorPath(workspacePath))) {
+      rankings.push(await sparseQuery({ workspacePath, config: config.retrieval.sparse, query: normalizedQuery, topK: candidateLimit }).then((sparse) => sparse.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit)));
+    }
+    hits = reciprocalRankFusion(rankings, { rankConstant: 20, weights: rankings.map((_, index) => index === 0 ? 3 : 1) }).slice(0, candidateLimit);
+  }
+  const rawResults = await Promise.all(hits.map(async ([chunkId, score]) => {
+    const chunk = chunks.get(chunkId);
+    if (!chunk) {
+      return null;
+    }
+    return {
+      chunkId,
+      documentId: chunk.documentId,
+      sourceId: chunk.sourceId,
+      sourceType: documents.get(chunk.documentId)?.sourceType ?? "text",
+      score,
+      title: chooseResultTitle(chunk),
+      uri: chunk.uri,
+      headingPath: chunk.headingPath,
+      snippet: await buildSnippetWithAdjacentChunks(chunk, normalizedQuery, {
+        document: documents.get(chunk.documentId),
+        config,
+        orderedChunkCache
+      }),
+      text: showChunks ? chunk.text : void 0,
+      publicationDate: documents.get(chunk.documentId)?.publicationDate ?? null,
+      firstSeenAt: documents.get(chunk.documentId)?.firstSeenAt ?? chunk.firstSeenAt,
+      lastSeenAt: documents.get(chunk.documentId)?.lastSeenAt ?? chunk.lastSeenAt,
+      lastChangedAt: documents.get(chunk.documentId)?.lastChangedAt ?? chunk.lastChangedAt,
+      metadata: chunk.metadata
+    };
+  }));
+  const results = rawResults.filter((result2) => result2 != null);
+  return { retrievalMode: mode, results: rerankResultsByDocument(results, topK) };
+}
+
+// src/query/related-service.ts
+import path19 from "path";
+function cosineSimilarity(left, right) {
+  let dot = 0;
+  let leftNorm = 0;
+  let rightNorm = 0;
+  for (let index = 0; index < left.length; index += 1) {
+    const leftValue = left[index] ?? 0;
+    const rightValue = right[index] ?? 0;
+    dot += leftValue * rightValue;
+    leftNorm += leftValue * leftValue;
+    rightNorm += rightValue * rightValue;
+  }
+  if (leftNorm === 0 || rightNorm === 0) {
+    return 0;
+  }
+  return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
+}
+function normalizeVector(values) {
+  const norm = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
+  if (norm === 0) {
+    return values.map(() => 0);
+  }
+  return values.map((value) => value / norm);
+}
+function averageEmbeddings(records, dimensions) {
+  const totals = new Array(dimensions).fill(0);
+  for (const record of records) {
+    for (let index = 0; index < dimensions; index += 1) {
+      totals[index] = (totals[index] ?? 0) + (record.embedding[index] ?? 0);
+    }
+  }
+  return normalizeVector(totals.map((value) => value / Math.max(records.length, 1)));
+}
+function resolveDocumentSelector(documents, selector) {
+  const normalized = selector.trim().toLowerCase();
+  const matches = documents.filter(
+    (document) => document.id.toLowerCase() === normalized || document.uri.toLowerCase() === normalized || document.canonicalUri?.toLowerCase() === normalized
+  );
+  if (matches.length === 0) {
+    throw new CliError(`document not found: ${selector}`, "DOCUMENT_NOT_FOUND", 2 /* InvalidArguments */);
+  }
+  if (matches.length > 1) {
+    throw new CliError(`document selector is ambiguous: ${selector}`, "DOCUMENT_SELECTOR_AMBIGUOUS", 2 /* InvalidArguments */);
+  }
+  return matches[0];
+}
+function buildDocumentVectors(documents, denseChunks, dimensions) {
+  const byDocument = /* @__PURE__ */ new Map();
+  for (const chunk of denseChunks) {
+    const existing = byDocument.get(chunk.documentId);
+    if (existing) {
+      existing.push(chunk);
+    } else {
+      byDocument.set(chunk.documentId, [chunk]);
+    }
+  }
+  return new Map(documents.flatMap((document) => {
+    const records = byDocument.get(document.id);
+    if (!records?.length) {
+      return [];
+    }
+    return [[document.id, { document, embedding: averageEmbeddings(records, dimensions) }]];
+  }));
+}
+async function findRelatedDocuments({
+  workspacePath,
+  document,
+  topK
+}) {
+  const config = await loadConfig(workspacePath);
+  if (!config.retrieval.dense.enabled) {
+    throw new CliError("dense retrieval is disabled in config; enable retrieval.dense.enabled and rebuild", "DENSE_RETRIEVAL_DISABLED", 7 /* QueryError */);
+  }
+  if (!await fileExists(denseVectorPath(workspacePath))) {
+    throw new CliError("dense vector index is not built; run `qli models pull --dense` and `qli rebuild`", "DENSE_INDEX_MISSING", 7 /* QueryError */);
+  }
+  const documents = await readJsonl(path19.join(workspacePath, "documents", "documents.jsonl"));
+  const selected = resolveDocumentSelector(documents, document);
+  const densePayload = await readDensePayload(workspacePath);
+  const vectors = buildDocumentVectors(documents, densePayload.chunks, densePayload.metadata.dimensions);
+  const sourceVector = vectors.get(selected.id);
+  if (!sourceVector) {
+    throw new CliError(`dense vectors are missing for document: ${document}`, "DOCUMENT_VECTOR_MISSING", 7 /* QueryError */);
+  }
+  const results = [...vectors.values()].filter((candidate) => candidate.document.id !== selected.id).map((candidate) => ({
+    documentId: candidate.document.id,
+    sourceId: candidate.document.sourceId,
+    score: cosineSimilarity(sourceVector.embedding, candidate.embedding),
+    title: candidate.document.title,
+    uri: candidate.document.uri,
+    metadata: candidate.document.metadata
+  })).sort((left, right) => right.score - left.score).slice(0, topK);
+  return {
+    sourceDocument: {
+      documentId: selected.id,
+      sourceId: selected.sourceId,
+      title: selected.title,
+      uri: selected.uri
+    },
+    retrievalMode: "dense",
+    results
+  };
+}
+
+// src/query/context-builder.ts
+async function createContext({
+  workspacePath,
+  query,
+  topK,
+  maxChars,
+  retrievalMode
+}) {
+  const search = await searchIndex({ workspacePath, query, topK, showChunks: true, retrievalMode });
+  const sources = [];
+  let total = 0;
+  for (const result2 of search.results) {
+    const text = result2.text ?? "";
+    if (total + text.length > maxChars && sources.length > 0) {
+      break;
+    }
+    total += text.length;
+    sources.push({
+      chunkId: result2.chunkId,
+      documentId: result2.documentId,
+      sourceId: result2.sourceId,
+      title: result2.title,
+      uri: result2.uri,
+      headingPath: result2.headingPath,
+      text,
+      metadata: result2.metadata
+    });
+  }
+  const markdown = [
+    "# Context",
+    "",
+    ...sources.flatMap((source, index) => [
+      `## Source ${index + 1}`,
+      `Title: ${source.title}`,
+      `URL: ${source.uri}`,
+      `Chunk ID: ${source.chunkId}`,
+      source.headingPath.length > 0 ? `Heading Path: ${source.headingPath.join(" > ")}` : "",
+      "",
+      source.text,
+      ""
+    ].filter((line) => line !== ""))
+  ].join("\n");
+  return { markdown, sources, retrievalMode: search.retrievalMode };
+}
+
+// src/report/diff-service.ts
+import path20 from "path";
+function chooseBaselineRun(runs, since) {
+  if (since === "last-run") {
+    return runs.at(-1);
+  }
+  if (since) {
+    return runs.filter((run) => run.createdAt < since).at(-1) ?? runs.at(-1);
+  }
+  return runs.at(-1);
+}
+async function diffWorkspace({
+  workspacePath,
+  sourceId,
+  documentId,
+  since
+}) {
+  const current = await readJsonl(path20.join(workspacePath, "documents", "documents.jsonl"));
+  const baseline = chooseBaselineRun(await listRuns(workspacePath), since);
+  const previous = new Map((baseline?.documentsSnapshot ?? []).map((document) => [document.id, document]));
+  const changedDocuments = current.filter((document) => (!sourceId || document.sourceId === sourceId) && (!documentId || document.id === documentId)).filter((document) => {
+    const prior = previous.get(document.id);
+    return !prior || prior.contentHash !== document.contentHash || since && document.lastChangedAt >= since;
+  }).map((document) => ({
+    id: document.id,
+    title: document.title,
+    uri: document.uri,
+    sourceId: document.sourceId,
+    previousHash: previous.get(document.id)?.contentHash,
+    currentHash: document.contentHash
+  }));
+  return { changedDocuments };
+}
+function renderChangeReport(diff) {
+  return [
+    "# Knowledge Base Change Report",
+    "",
+    "## Summary",
+    "",
+    `Changed documents: ${diff.changedDocuments.length}`,
+    "",
+    "## Added Documents",
+    "",
+    "_No added documents in this simple report._",
+    "",
+    "## Changed Documents",
+    "",
+    ...diff.changedDocuments.map((document) => `- ${document.title} (${document.uri}) [${document.id}]`),
+    "",
+    "## Removed or Missing Documents",
+    "",
+    "_Removal tracking is not available for this report._",
+    "",
+    "## Notable Changed Sections",
+    "",
+    ...diff.changedDocuments.map((document) => `- ${document.sourceId}: ${document.title}`)
+  ].join("\n");
+}
+
+// src/cli/format.ts
+import Table from "cli-table3";
+import colors from "picocolors";
+function formatSourcesTable(sources) {
+  const table = new Table({ head: ["ID", "TYPE", "NAME", "URI", "ENABLED", "TAGS"] });
+  for (const source of sources) {
+    table.push([source.id, source.type, source.name, source.uri, String(source.enabled), source.tags.join(",")]);
+  }
+  return table.toString();
+}
+function formatSearchResults(results) {
+  return results.map((result2, index) => [
+    `${index + 1}. ${colors.bold(result2.title)}`,
+    `   ${result2.uri}`,
+    `   Source type: ${result2.sourceType}`,
+    `   Published: ${result2.publicationDate ?? "n/a"}`,
+    `   Score: ${result2.score.toFixed(3)}`,
+    `   ${result2.snippet}`
+  ].join("\n")).join("\n\n");
+}
+function formatRelatedDocuments(results) {
+  return results.map((result2, index) => [
+    `${index + 1}. ${colors.bold(result2.title)}`,
+    `   ${result2.uri}`,
+    `   Similarity: ${result2.score.toFixed(3)}`
+  ].join("\n")).join("\n\n");
+}
+
+// src/cli/run-cli.ts
+var SOURCE_TYPES = /* @__PURE__ */ new Set(["url", "website", "rss", "file", "directory", "markdown", "text"]);
+var RETRIEVAL_MODES = /* @__PURE__ */ new Set(["lexical", "dense", "sparse", "hybrid"]);
+var SOURCE_TYPE_LIST = ["url", "website", "rss", "file", "directory", "markdown", "text"];
+var RETRIEVAL_MODE_LIST = ["lexical", "dense", "sparse", "hybrid"];
+var SEARCH_DATE_FIELDS = ["publicationDate", "firstSeenAt", "lastSeenAt", "lastChangedAt", "crawledAt"];
+function parseKeyValue(input) {
+  const idx = input.indexOf("=");
+  if (idx <= 0) {
+    throw new CliError(`invalid key=value pair: ${input}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+  }
+  return [input.slice(0, idx), input.slice(idx + 1)];
+}
+function normalizeMetadata(values = []) {
+  return Object.fromEntries(values.map(parseKeyValue));
+}
+function parseOptionalNumber(input, optionName) {
+  if (input === void 0) {
+    return void 0;
+  }
+  const value = Number(input);
+  if (!Number.isFinite(value)) {
+    throw new CliError(`invalid number for ${optionName}: ${input}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+  }
+  return value;
+}
+function setWhenDefined(target, key, value) {
+  if (value !== void 0) {
+    target[key] = value;
+  }
+}
+function createSourceCrawlConfig(type, options, defaults) {
+  if (!["url", "website", "directory", "rss"].includes(type)) {
+    return void 0;
+  }
+  const crawl = {};
+  setWhenDefined(crawl, "maxDepth", parseOptionalNumber(options.maxDepth, "--max-depth"));
+  setWhenDefined(crawl, "maxPages", parseOptionalNumber(options.maxPages, "--max-pages"));
+  setWhenDefined(crawl, "includePatterns", options.include);
+  setWhenDefined(crawl, "excludePatterns", options.exclude);
+  setWhenDefined(crawl, "obeyRobotsTxt", options.robots);
+  setWhenDefined(crawl, "rateLimitMs", parseOptionalNumber(options.rateLimitMs, "--rate-limit-ms"));
+  if (options.renderJs) {
+    crawl.renderJs = true;
+  }
+  if (type === "website") {
+    crawl.useSitemap = true;
+  }
+  if (type === "rss") {
+    crawl.retentionDays = parseOptionalNumber(options.retentionDays, "--retention-days") ?? defaults.retentionDays;
+    crawl.fetchArticles = true;
+  } else {
+    setWhenDefined(crawl, "retentionDays", parseOptionalNumber(options.retentionDays, "--retention-days"));
+  }
+  return Object.keys(crawl).length > 0 ? crawl : void 0;
+}
+function allowedSourceConfigFields(source) {
+  const fields = /* @__PURE__ */ new Set(["name", "tag", "metadata"]);
+  if (source.type === "rss") {
+    fields.add("retentionDays");
+  }
+  if (source.type === "website") {
+    fields.add("maxDepth");
+    fields.add("maxPages");
+    fields.add("include");
+    fields.add("exclude");
+  }
+  if (source.type === "directory") {
+    fields.add("include");
+    fields.add("exclude");
+  }
+  return fields;
+}
+function buildSourceConfigPatch(source, options) {
+  const allowed = allowedSourceConfigFields(source);
+  const patch = {};
+  if (options.name !== void 0) {
+    patch.name = options.name;
+  }
+  if (options.tag !== void 0) {
+    patch.tags = options.tag;
+  }
+  if (options.metadata !== void 0) {
+    patch.metadata = normalizeMetadata(options.metadata);
+  }
+  const crawlPatch = {};
+  const checkAllowed = (field, optionName) => {
+    if (!allowed.has(field)) {
+      throw new CliError(`${optionName} is not supported for source type ${source.type}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+    }
+  };
+  if (options.maxDepth !== void 0) {
+    checkAllowed("maxDepth", "--max-depth");
+    crawlPatch.maxDepth = parseOptionalNumber(options.maxDepth, "--max-depth");
+  }
+  if (options.maxPages !== void 0) {
+    checkAllowed("maxPages", "--max-pages");
+    crawlPatch.maxPages = parseOptionalNumber(options.maxPages, "--max-pages");
+  }
+  if (options.include !== void 0) {
+    checkAllowed("include", "--include");
+    crawlPatch.includePatterns = options.include;
+  }
+  if (options.exclude !== void 0) {
+    checkAllowed("exclude", "--exclude");
+    crawlPatch.excludePatterns = options.exclude;
+  }
+  if (options.retentionDays !== void 0) {
+    checkAllowed("retentionDays", "--retention-days");
+    crawlPatch.retentionDays = parseOptionalNumber(options.retentionDays, "--retention-days");
+  }
+  if (Object.keys(crawlPatch).length > 0) {
+    patch.crawl = crawlPatch;
+  }
+  return patch;
+}
+function response(command, workspace, data, error) {
+  return {
+    ok: !error,
+    command,
+    workspace,
+    version: PACKAGE_VERSION,
+    data,
+    error
+  };
+}
+function writeOutput(capture, value, stderr = false) {
+  (stderr ? capture.stderr : capture.stdout).push(value);
+}
+function parseRetrievalMode(input) {
+  if (!input) {
+    return void 0;
+  }
+  if (!RETRIEVAL_MODES.has(input)) {
+    throw new CliError(`unsupported retrieval mode: ${input}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+  }
+  return input;
+}
+function parseSourceType(input) {
+  if (!input) {
+    return void 0;
+  }
+  if (!SOURCE_TYPES.has(input)) {
+    throw new CliError(`unsupported source type: ${input}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+  }
+  return input;
+}
+function parseCommaSeparatedList(input) {
+  const values = (input ?? "").split(",").map((value) => value.trim()).filter(Boolean);
+  return values.length > 0 ? values : void 0;
+}
+function parseSourceTypes(input) {
+  const values = parseCommaSeparatedList(input);
+  if (!values) {
+    return void 0;
+  }
+  return values.map((value) => parseSourceType(value));
+}
+function parseDateValue(input, optionName) {
+  const parsed = new Date(input);
+  if (Number.isNaN(parsed.getTime())) {
+    throw new CliError(`invalid date for ${optionName}: ${input}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+  }
+  return parsed.toISOString();
+}
+function searchDateRanges(options) {
+  const entries = [];
+  if (options.since || options.until) {
+    entries.push({
+      field: "publicationDate",
+      from: options.since ? parseDateValue(options.since, "--since") : void 0,
+      to: options.until ? parseDateValue(options.until, "--until") : void 0
+    });
+  }
+  if (options.changedSince) {
+    entries.push({
+      field: "lastChangedAt",
+      from: parseDateValue(options.changedSince, "--changed-since")
+    });
+  }
+  for (const field of SEARCH_DATE_FIELDS) {
+    const fromKey = `${field}From`;
+    const toKey = `${field}To`;
+    const from = options[fromKey];
+    const to = options[toKey];
+    if (!from && !to) {
+      continue;
+    }
+    entries.push({
+      field,
+      from: from ? parseDateValue(from, `--${field}-from`) : void 0,
+      to: to ? parseDateValue(to, `--${field}-to`) : void 0
+    });
+  }
+  return entries;
+}
+async function resolveWorkspace(options) {
+  return path21.resolve(options.workspace ?? DEFAULT_WORKSPACE);
+}
+function workspaceFromArgv(argv) {
+  const index = argv.findIndex((arg) => arg === "--workspace");
+  if (index >= 0 && argv[index + 1]) {
+    return path21.resolve(argv[index + 1]);
+  }
+  return path21.resolve(DEFAULT_WORKSPACE);
+}
+async function runCli(argv) {
+  const capture = { stdout: [], stderr: [] };
+  const program = new Command();
+  program.name("qli").description("Build and query a local Querylight workspace from files, directories, URLs, websites, and feeds.").showHelpAfterError().option("--workspace <path>", "Workspace directory. Defaults to .kb in the current directory.", DEFAULT_WORKSPACE).option("--config <path>", "Optional config file override. Useful for testing alternate retrieval settings.").option("--json", "Return a stable JSON envelope for automation and agents.").option("--verbose", "Print more operational detail when a command supports it.").option("--quiet", "Suppress non-essential human-readable output.");
+  program.addHelpText("after", `
+Workflow:
+  1. Initialize a workspace with qli init
+  2. Register one or more sources with qli source add
+  3. Build or refresh the workspace with qli rebuild
+  4. Query it with qli search, qli related, or qli context
+
+Examples:
+  qli init
+  qli source add directory ./docs --name "Product Docs" --tag docs
+  qli rebuild
+  qli search "api authentication" --top-k 8
+  qli context "How do API keys work?" --top-k 8 --max-chars 8000
+
+Use qli <command> --help for command-specific options and examples.`);
+  program.command("init").description("Create a new workspace with the default directory layout and config.").option("--force").addHelpText("after", `
+Examples:
+  qli init
+  qli init --workspace ./kb
+  qli init --workspace /tmp/querylight --force`).action(async function command(options) {
+    const workspace = await resolveWorkspace({ workspace: this.optsWithGlobals().workspace });
+    const result2 = await ensureWorkspace({ workspacePath: workspace, force: Boolean(options.force) });
+    emit(this.optsWithGlobals().json, capture, response("init", workspace, result2), `Initialized workspace at ${workspace}`);
+  });
+  const source = program.command("source");
+  source.description("Register, inspect, and manage workspace sources.");
+  source.command("add").description("Add a source definition. The source is enabled immediately.").argument("<type>", `Source type: ${SOURCE_TYPE_LIST.join(", ")}`).argument("<uri>", "Local path, URL, feed URL, or inline content depending on the source type.").requiredOption("--name <name>").option("--tag <tag...>", "Optional tags used later for filtering during search.").option("--metadata <key=value...>", "Extra metadata fields stored on the source.").option("--max-depth <n>", "Maximum crawl depth for website sources.").option("--max-pages <n>", "Maximum number of pages to ingest from a website source.").option("--include <pattern...>", "Only include matching paths or URLs.").option("--exclude <pattern...>", "Skip matching paths or URLs.").option("--render-js", "Render pages with JavaScript before extraction when supported.").option("--no-robots", "Ignore robots.txt for website crawling. Use only when you control the target site or have permission.").option("--rate-limit-ms <n>", "Delay between website requests.").option("--retention-days <n>", "Retention window in days for RSS items. Defaults to the workspace crawler retention setting.").addHelpText("after", `
+Examples:
+  qli source add directory ./docs --name "Local Docs" --tag docs
+  qli source add file ./docs/auth.md --name "Auth Guide"
+  qli source add url https://example.com/docs/auth --name "Auth Page"
+  qli source add website https://example.com --name "Docs Site" --max-depth 2 --max-pages 50 --include /docs/
+  qli source add rss https://example.com/feed.xml --name "Release Feed"
+  qli source add rss https://example.com/feed.xml --name "Release Feed" --retention-days 30
+
+Notes:
+  RSS sources store retention per feed.
+  When you omit --retention-days for RSS, qli stores the workspace default from config.yaml.`).action(async function command(type, uri, options) {
+    if (!SOURCE_TYPES.has(type)) {
+      throw new CliError(`unsupported source type: ${type}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+    }
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const config = await loadConfig(workspace, global.config);
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const crawl = createSourceCrawlConfig(type, options, { retentionDays: config.crawler.retentionDays });
+    const stored = await addSource(workspace, {
+      type,
+      uri: ["file", "directory"].includes(type) ? path21.resolve(uri) : uri,
+      name: options.name,
+      enabled: true,
+      tags: options.tag ?? [],
+      metadata: normalizeMetadata(options.metadata),
+      crawl,
+      createdAt: now,
+      updatedAt: now
+    });
+    emit(global.json, capture, response("source add", workspace, stored), `Added source ${stored.id}`);
+  });
+  source.command("config").description("Edit supported settings on an existing source.").argument("<sourceId>", "Source id from qli source list.").option("--name <name>", "Update the source name.").option("--tag <tag...>", "Replace source tags with the provided values.").option("--metadata <key=value...>", "Merge metadata keys into the existing source metadata.").option("--max-depth <n>", "Set website crawl depth.").option("--max-pages <n>", "Set the page limit for website sources.").option("--include <pattern...>", "Set include patterns for website or directory sources.").option("--exclude <pattern...>", "Set exclude patterns for website or directory sources.").option("--retention-days <n>", "Set RSS retention in days for this feed.").addHelpText("after", `
+Examples:
+  qli source config src_123 --retention-days 30
+  qli source config src_123 --name "Docs Feed" --tag rss docs
+  qli source config src_123 --include /docs/ --exclude /docs/archive/
+  qli source config src_123 --metadata team=docs owner=platform --json
+
+Notes:
+  qli only exposes settings that the current source type uses at runtime.
+  URI, source type, and source id do not change here.`).action(async function command(sourceId, options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const sources = await listSources(workspace);
+    const current = sources.find((source2) => source2.id === sourceId);
+    if (!current) {
+      throw new CliError(`source not found: ${sourceId}`, "SOURCE_NOT_FOUND", 4 /* SourceError */);
+    }
+    const patch = buildSourceConfigPatch(current, options);
+    if (Object.keys(patch).length === 0) {
+      throw new CliError("no changes requested", "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+    }
+    patch.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
+    const updated = await updateSource(workspace, sourceId, patch);
+    emit(global.json, capture, response("source config", workspace, updated), `Updated source ${sourceId}`);
+  });
+  source.command("list").description("List all configured sources in the workspace.").addHelpText("after", `
+Examples:
+  qli source list
+  qli source list --json`).action(async function command() {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const sources = await listSources(workspace);
+    emit(global.json, capture, response("source list", workspace, sources), formatSourcesTable(sources));
+  });
+  source.command("remove").description("Delete a source definition from the workspace.").argument("<sourceId>", "Source id from qli source list.").addHelpText("after", `
+Examples:
+  qli source remove src_123
+  qli source list --json`).action(async function command(sourceId) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    await removeSource(workspace, sourceId);
+    emit(global.json, capture, response("source remove", workspace, { sourceId }), `Removed source ${sourceId}`);
+  });
+  source.command("disable").description("Disable a source without removing its configuration.").argument("<sourceId>", "Source id from qli source list.").addHelpText("after", `
+Examples:
+  qli source disable src_123
+  qli source enable src_123`).action(async function command(sourceId) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const updated = await updateSource(workspace, sourceId, { enabled: false, updatedAt: (/* @__PURE__ */ new Date()).toISOString() });
+    emit(global.json, capture, response("source disable", workspace, updated), `Disabled source ${sourceId}`);
+  });
+  source.command("enable").description("Re-enable a disabled source.").argument("<sourceId>", "Source id from qli source list.").addHelpText("after", `
+Examples:
+  qli source enable src_123
+  qli source list`).action(async function command(sourceId) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const updated = await updateSource(workspace, sourceId, { enabled: true, updatedAt: (/* @__PURE__ */ new Date()).toISOString() });
+    emit(global.json, capture, response("source enable", workspace, updated), `Enabled source ${sourceId}`);
+  });
+  program.command("ingest").description("Fetch and normalize source content into workspace documents.").option("--source <sourceId>", "Only ingest one source.").option("--changed-only", "Skip content that has not changed since the last run.").addHelpText("after", `
+Examples:
+  qli ingest
+  qli ingest --source src_123
+  qli ingest --changed-only`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await ingestSources({ workspacePath: workspace, sourceIds: options.source ? [options.source] : void 0, changedOnly: Boolean(options.changedOnly) });
+    emit(global.json, capture, response("ingest", workspace, result2), `Ingested ${result2.processedSources} sources`);
+  });
+  program.command("chunk").description("Split normalized documents into retrieval chunks.").option("--source <sourceId>", "Only chunk documents from one source.").option("--document <documentId>", "Only chunk one document.").addHelpText("after", `
+Examples:
+  qli chunk
+  qli chunk --source src_123
+  qli chunk --document doc_123`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await chunkDocuments({ workspacePath: workspace, sourceId: options.source, documentId: options.document });
+    emit(global.json, capture, response("chunk", workspace, result2), `Wrote ${result2.chunksWritten} chunks`);
+  });
+  program.command("reprocess").description("Re-run normalization for existing documents without fetching sources again.").option("--source <sourceId>", "Only reprocess documents from one source.").option("--document <documentId>", "Only reprocess one document.").addHelpText("after", `
+Examples:
+  qli reprocess
+  qli reprocess --source src_123
+  qli reprocess --document doc_123`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await reprocessDocuments({ workspacePath: workspace, sourceId: options.source, documentId: options.document });
+    emit(global.json, capture, response("reprocess", workspace, result2), `Reprocessed ${result2.documentsReprocessed} documents`);
+  });
+  const index = program.command("index");
+  index.description("Build and inspect retrieval indexes.");
+  index.command("build").description("Build lexical search artifacts and optional dense or sparse vector indexes.").option("--dense", "Force a dense vector build if the dense model is available.").option("--sparse", "Force a sparse vector build if the sparse runtime is available.").addHelpText("after", `
+Examples:
+  qli index build
+  qli index build --dense
+  qli index build --dense --sparse`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await buildIndex({
+      workspacePath: workspace,
+      denseOverride: options.dense ? true : void 0,
+      sparseOverride: options.sparse ? true : void 0
+    });
+    emit(global.json, capture, response("index build", workspace, result2), `Built index at ${result2.indexPath}`);
+  });
+  program.command("rebuild").description("Run ingest, chunk, and index build in one command.").option("--source <sourceId>", "Only rebuild data for one source.").option("--changed-only", "Only ingest changed content before chunking and indexing.").option("--dense", "Force a dense vector build if the dense model is available.").option("--sparse", "Force a sparse vector build if the sparse runtime is available.").addHelpText("after", `
+Examples:
+  qli rebuild
+  qli rebuild --changed-only
+  qli rebuild --source src_123
+  qli rebuild --dense --sparse`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const ingest = await ingestSources({ workspacePath: workspace, sourceIds: options.source ? [options.source] : void 0, changedOnly: Boolean(options.changedOnly) });
+    const chunk = await chunkDocuments({ workspacePath: workspace, sourceId: options.source });
+    const indexBuild = await buildIndex({
+      workspacePath: workspace,
+      denseOverride: options.dense ? true : void 0,
+      sparseOverride: options.sparse ? true : void 0,
+      buildAvailableModels: true
+    });
+    const data = { ingest, chunk, indexPath: indexBuild.indexPath, metadata: indexBuild.metadata };
+    emit(global.json, capture, response("rebuild", workspace, data), `Processed ${ingest.processedSources} sources, wrote ${chunk.chunksWritten} chunks`);
+  });
+  program.command("search").description("Search the built index and return ranked matching documents or chunks.").argument("[query]", "Text query. Omit it to list the latest matching documents.").option("--top-k <n>", "Maximum number of results to return.", "12").option("--source <sourceIds>", "Restrict results to one or more source ids. Use comma-separated values.").option("--source-name <names>", "Restrict results to one or more source names. Use comma-separated values.").option("--source-type <types>", `Restrict results to one or more source types. Use comma-separated values: ${SOURCE_TYPE_LIST.join(", ")}`).option("--uri-prefix <prefixes>", "Restrict results to one or more URI prefixes. Use comma-separated values.").option("--tag <tags>", "Restrict results to one or more source tags. Use comma-separated values.").option("--metadata <key=value...>", "Restrict results to sources with matching metadata.").option("--since <date>", "Shortcut for --publication-date-from.").option("--until <date>", "Shortcut for --publication-date-to.").option("--changed-since <date>", "Only include documents changed on or after this date.").option("--has-publication-date", "Only include documents with a publication date.").option("--publication-date-from <date>", "Only include documents published on or after this date.").option("--publication-date-to <date>", "Only include documents published on or before this date.").option("--first-seen-at-from <date>", "Only include documents first seen on or after this date.").option("--first-seen-at-to <date>", "Only include documents first seen on or before this date.").option("--last-seen-at-from <date>", "Only include documents last seen on or after this date.").option("--last-seen-at-to <date>", "Only include documents last seen on or before this date.").option("--last-changed-at-from <date>", "Only include documents changed on or after this date.").option("--last-changed-at-to <date>", "Only include documents changed on or before this date.").option("--crawled-at-from <date>", "Only include documents crawled on or after this date.").option("--crawled-at-to <date>", "Only include documents crawled on or before this date.").option("--retrieval <mode>", `Retrieval mode: ${RETRIEVAL_MODE_LIST.join(", ")}`).option("--show-chunks", "Return chunk-level matches when available.").addHelpText("after", `
+Examples:
+  qli search "pricing api limits"
+  qli search "authentication" --top-k 20 --tag docs
+  qli search --source-type rss --since 2026-05-01 --has-publication-date
+  qli search --source-name "Release Feed,Company Blog" --uri-prefix https://example.com/news,https://example.com/blog
+  qli search "billing" --metadata team=support
+  qli search "embedding model" --retrieval hybrid --show-chunks
+  qli search --source-type rss,url --top-k 25 --json
+
+Notes:
+  lexical works without vector models.
+  dense, sparse, and hybrid require the relevant index artifacts to exist.
+  When you omit the query, qli returns the latest matching documents sorted by publication date.`).action(async function command(query, options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await searchIndex({
+      workspacePath: workspace,
+      query: query ?? "",
+      topK: Number(options.topK),
+      sourceIds: parseCommaSeparatedList(options.source),
+      sourceNames: parseCommaSeparatedList(options.sourceName),
+      sourceTypes: parseSourceTypes(options.sourceType),
+      uriPrefixes: parseCommaSeparatedList(options.uriPrefix),
+      hasPublicationDate: Boolean(options.hasPublicationDate),
+      tags: parseCommaSeparatedList(options.tag),
+      metadata: (options.metadata ?? []).map(parseKeyValue).map(([key, value]) => ({ key, value })),
+      dateRanges: searchDateRanges(options),
+      retrievalMode: parseRetrievalMode(options.retrieval),
+      showChunks: Boolean(options.showChunks)
+    });
+    emit(global.json, capture, response("search", workspace, result2), formatSearchResults(result2.results));
+  });
+  program.command("related").description("Find documents similar to an existing document by id or URI.").argument("<document>", "Document id, uri, or canonical uri").option("--top-k <n>", "Maximum number of related documents to return.", "12").addHelpText("after", `
+Examples:
+  qli related doc_123
+  qli related https://example.com/docs/auth
+
+Dense vectors usually produce better related-document results. Pull models and rebuild first when needed:
+  qli models pull --dense
+  qli rebuild --dense`).action(async function command(document, options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await findRelatedDocuments({
+      workspacePath: workspace,
+      document,
+      topK: Number(options.topK)
+    });
+    emit(global.json, capture, response("related", workspace, result2), formatRelatedDocuments(result2.results));
+  });
+  program.command("context").description("Assemble retrieval context for an external LLM, agent, or prompt pipeline.").argument("<query>").option("--top-k <n>", "Maximum number of source passages to consider.", "12").option("--max-chars <n>", "Maximum output length for the rendered context block.", "12000").option("--retrieval <mode>", `Retrieval mode: ${RETRIEVAL_MODE_LIST.join(", ")}`).addHelpText("after", `
+Examples:
+  qli context "How do I configure the API?"
+  qli context "What changed in pricing?" --top-k 10 --max-chars 9000
+  qli context "How does auth work?" --retrieval hybrid
+
+Use --json when another tool needs structured access to the raw passages and metadata.`).action(async function command(query, options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await createContext({
+      workspacePath: workspace,
+      query,
+      topK: Number(options.topK),
+      maxChars: Number(options.maxChars),
+      retrievalMode: parseRetrievalMode(options.retrieval)
+    });
+    emit(global.json, capture, response("context", workspace, result2), result2.markdown);
+  });
+  const models = program.command("models");
+  models.description("Inspect and download retrieval model assets.");
+  models.command("pull").description("Download dense and or sparse retrieval assets required by vector search.").option("--dense", "Only pull dense retrieval assets.").option("--sparse", "Only pull sparse retrieval assets.").addHelpText("after", `
+Examples:
+  qli models pull
+  qli models pull --dense
+  qli models pull --sparse
+
+If you plan to use related, dense search, or hybrid retrieval, pull the models and rebuild the index first.`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const config = await loadConfig(workspace, global.config);
+    const status = await getModelStatus(workspace, config);
+    const { pullDense, pullSparse } = resolveModelPullPlan({
+      pullDenseFlag: Boolean(options.dense),
+      pullSparseFlag: Boolean(options.sparse),
+      uvAvailable: status.sparse.uvAvailable
+    });
+    await pullModels({ workspacePath: workspace, config, pullDense, pullSparse });
+    const data = {
+      dense: pullDense ? { pulled: true, modelId: config.retrieval.dense.modelId, cacheDir: config.retrieval.dense.cacheDir } : void 0,
+      sparse: pullSparse ? { pulled: true, modelId: config.retrieval.sparse.modelId, cacheDir: config.retrieval.sparse.cacheDir } : void 0
+    };
+    emit(global.json, capture, response("models pull", workspace, data), "Pulled available models");
+  });
+  models.command("status").description("Show whether model runtimes and artifacts are available in the workspace.").addHelpText("after", `
+Examples:
+  qli models status
+  qli models status --json`).action(async function command() {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const config = await loadConfig(workspace, global.config);
+    const data = await getModelStatus(workspace, config);
+    emit(global.json, capture, response("models status", workspace, data), JSON.stringify(data, null, 2));
+  });
+  program.command("diff").description("Inspect document-level changes between stored workspace versions.").option("--source <sourceId>", "Only inspect changes for one source.").option("--document <documentId>", "Only inspect one document.").option("--since <timestamp>", "Only include changes since an ISO timestamp.").addHelpText("after", `
+Examples:
+  qli diff
+  qli diff --source src_123
+  qli diff --document doc_123
+  qli diff --since 2026-05-01`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const result2 = await diffWorkspace({ workspacePath: workspace, sourceId: options.source, documentId: options.document, since: options.since });
+    emit(global.json, capture, response("diff", workspace, result2), JSON.stringify(result2, null, 2));
+  });
+  const report = program.command("report");
+  report.description("Render higher-level reports from workspace data.");
+  report.command("changes").description("Render a markdown change report from workspace diffs.").option("--source <sourceId>", "Only include one source.").option("--since <timestamp>", "Only include changes since an ISO timestamp.").addHelpText("after", `
+Examples:
+  qli report changes
+  qli report changes --since 2026-05-01
+  qli report changes --source src_123 --json`).action(async function command(options) {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const diff = await diffWorkspace({ workspacePath: workspace, sourceId: options.source, since: options.since });
+    const markdown = renderChangeReport(diff);
+    emit(global.json, capture, response("report changes", workspace, { markdown, diff }), markdown);
+  });
+  program.command("status").description("Summarize workspace size, index state, and model artifact availability.").addHelpText("after", `
+Examples:
+  qli status
+  qli status --json`).action(async function command() {
+    const global = this.optsWithGlobals();
+    const workspace = await resolveWorkspace({ workspace: global.workspace });
+    const sources = await listSources(workspace);
+    const documents = await readJsonl(`${workspace}/documents/documents.jsonl`);
+    const chunks = await readJsonl(`${workspace}/chunks/chunks.jsonl`);
+    const runs = await listRuns(workspace);
+    const config = await loadConfig(workspace, global.config);
+    const modelStatus = await getModelStatus(workspace, config);
+    let latestIndex;
+    let indexSize = 0;
+    try {
+      const meta = await readLatestIndexMetadata(workspace);
+      latestIndex = meta.createdAt;
+      indexSize = (await stat4(`${workspace}/indexes/latest.json`)).size;
+    } catch {
+      latestIndex = void 0;
+    }
+    const data = {
+      workspace,
+      sources: sources.length,
+      documents: documents.length,
+      chunks: chunks.length,
+      latestIndex,
+      indexSizeBytes: indexSize,
+      lastRun: runs.at(-1)?.success ? "success" : runs.at(-1) ? "failed" : "none",
+      denseVectorIndex: modelStatus.dense.artifactExists,
+      sparseVectorIndex: modelStatus.sparse.artifactExists
+    };
+    emit(global.json, capture, response("status", workspace, data), [
+      `Workspace: ${workspace}`,
+      `Sources: ${data.sources}`,
+      `Documents: ${data.documents}`,
+      `Chunks: ${data.chunks}`,
+      `Latest index: ${data.latestIndex ?? "none"}`,
+      `Index size: ${Math.round(indexSize / 1024)} KB`,
+      `Last run: ${data.lastRun}`,
+      `Dense vector index: ${data.denseVectorIndex}`,
+      `Sparse vector index: ${data.sparseVectorIndex}`
+    ].join("\n"));
+  });
+  program.command("doctor").description("Run basic workspace and runtime checks.").addHelpText("after", `
+Examples:
+  qli doctor
+  qli doctor --json`).action(async function command() {
+    const global = this.optsWithGlobals();
+    const workspace = await assertWorkspaceExists(await resolveWorkspace({ workspace: global.workspace }));
+    const checks = [];
+    await loadConfig(workspace, global.config);
+    checks.push("workspace exists");
+    checks.push("config parses");
+    await listSources(workspace);
+    checks.push("sources parse");
+    await readJsonl(`${workspace}/documents/documents.jsonl`);
+    checks.push("documents parse");
+    await readJsonl(`${workspace}/chunks/chunks.jsonl`);
+    checks.push("chunks parse");
+    const config = await loadConfig(workspace, global.config);
+    if (config.retrieval.dense.enabled) {
+      await import("@huggingface/transformers");
+      checks.push("dense runtime importable");
+    }
+    if (config.retrieval.sparse.enabled) {
+      await ensureUvAvailable();
+      checks.push("uv available for sparse runtime");
+    }
+    try {
+      await readLatestIndexMetadata(workspace);
+      checks.push("latest index exists");
+    } catch {
+      checks.push("latest index missing");
+    }
+    emit(global.json, capture, response("doctor", workspace, { checks }), checks.join("\n"));
+  });
+  let exitCode = 0;
+  try {
+    await program.parseAsync(["node", "qli", ...argv], { from: "node" });
+  } catch (error) {
+    const workspace = workspaceFromArgv(argv);
+    const cliError = error instanceof CliError ? error : new CliError(error.message, "GENERAL_ERROR", 1 /* GeneralError */);
+    writeOutput(capture, JSON.stringify(response("error", workspace, void 0, {
+      code: cliError.code,
+      message: cliError.message,
+      details: cliError.details
+    })), true);
+    exitCode = cliError.exitCode;
+  }
+  return {
+    exitCode,
+    stdout: capture.stdout.join("\n"),
+    stderr: capture.stderr.join("\n")
+  };
+}
+function emit(asJson, capture, body, human) {
+  writeOutput(capture, asJson ? JSON.stringify(body) : human);
+}
+
+// src/cli/main.ts
+var result = await runCli(process.argv.slice(2));
+if (result.stdout) {
+  process.stdout.write(`${result.stdout}
+`);
+}
+if (result.stderr) {
+  process.stderr.write(`${result.stderr}
+`);
+}
+process.exit(result.exitCode);