@tryformation/querylight-cli 0.2.2 → 0.2.4

package/dist/index.js

@@ -57,7 +57,7 @@ var defaultConfig = () => ({
     defaultMode: "lexical",
     dense: {
       enabled: true,
-      modelId: "Xenova/all-MiniLM-L6-v2",
+      modelId: "Xenova/paraphrase-MiniLM-L3-v2",
       cacheDir: DEFAULT_SHARED_MODEL_CACHE_DIR,
       indexHashTables: 8,
       indexRandomSeed: 42,
@@ -65,7 +65,7 @@ var defaultConfig = () => ({
     },
     sparse: {
       enabled: true,
-      modelId: "opensearch-project/opensearch-neural-sparse-encoding-doc-v3-distill",
+      modelId: "opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini",
       cacheDir: DEFAULT_SHARED_MODEL_CACHE_DIR,
       documentTopTokens: 128,
       queryEncoding: "tokenizer-token-weights",
@@ -1213,13 +1213,17 @@ function isAllowed(url, baseUrl, includePatterns, excludePatterns, disallowRules
   if (url.search.length > 0) {
     return false;
   }
-  if (url.pathname.endsWith(".xml")) {
+  const pathname = url.pathname.toLowerCase();
+  if (pathname.endsWith(".xml")) {
     return false;
   }
-  if (url.pathname.includes("/cdn-cgi/")) {
+  if (pathname.endsWith(".pdf")) {
     return false;
   }
-  if (url.pathname === "/search" || url.pathname === "/search/" || url.pathname.endsWith("/search/")) {
+  if (pathname.includes("/cdn-cgi/")) {
+    return false;
+  }
+  if (pathname === "/search" || pathname === "/search/" || pathname.endsWith("/search/")) {
     return false;
   }
   if (disallowRules.some((rule) => rule !== "/" && url.pathname.startsWith(rule))) {
@@ -1782,7 +1786,7 @@ async function chunkDocuments({
 }
 
 // src/index/querylight-indexer.ts
-import { Analyzer, DocumentIndex, KeywordTokenizer, LowerCaseTextFilter, RankingAlgorithm, TextFieldIndex } from "@tryformation/querylight-ts";
+import { Analyzer, DateFieldIndex, DocumentIndex, KeywordTokenizer, LowerCaseTextFilter, RankingAlgorithm, StoredSourceIndex, TextFieldIndex } from "@tryformation/querylight-ts";
 import path17 from "path";
 
 // src/vector/dense.ts
@@ -2058,15 +2062,26 @@ function createSparseChunkText(chunk) {
 // src/vector/dense.ts
 var denseEmbedderFactory = null;
 var EXACT_DENSE_RERANK_THRESHOLD = 5e3;
+function normalizeDenseEmbedder(embedder) {
+  if (typeof embedder === "function") {
+    return { embed: embedder };
+  }
+  return embedder;
+}
 async function createEmbedder(cacheDir, modelId) {
   if (denseEmbedderFactory) {
-    return denseEmbedderFactory(cacheDir, modelId);
+    return normalizeDenseEmbedder(await denseEmbedderFactory(cacheDir, modelId));
   }
   const runtime = await getDenseTransformersRuntime(cacheDir);
   const extractor = await runtime.pipeline("feature-extraction", modelId);
-  return async (text) => {
-    const output = await extractor(text, { pooling: "mean", normalize: true });
-    return output.tolist()[0];
+  return {
+    async embed(text) {
+      const output = await extractor(text, { pooling: "mean", normalize: true });
+      return output.tolist()[0];
+    },
+    async dispose() {
+      await extractor.dispose();
+    }
   };
 }
 function exactDenseQuery(payload, vector, topK) {
@@ -2080,53 +2095,57 @@ async function buildDenseVectors({
   const chunks = await readJsonl(path14.join(workspacePath, "chunks", "chunks.jsonl"));
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
   await mkdir7(cacheDir, { recursive: true });
-  const embed = await createEmbedder(cacheDir, config.modelId);
-  const records = [];
-  let dimensions = 0;
-  reportProgress(progress, `Encoding ${chunks.length} chunk${chunks.length === 1 ? "" : "s"} for dense retrieval`);
-  for (const chunk of chunks) {
-    const embedding = await embed(createDenseChunkText(chunk));
-    dimensions ||= embedding.length;
-    records.push({
-      chunkId: chunk.id,
-      documentId: chunk.documentId,
-      sourceId: chunk.sourceId,
-      title: chunk.title,
-      uri: chunk.uri,
-      headingPath: chunk.headingPath,
-      text: chunk.text,
-      embedding
-    });
-    if (records.length === 1 || records.length % 100 === 0 || records.length === chunks.length) {
-      reportProgressDetail(progress, `Encoded ${records.length}/${chunks.length} chunks for dense retrieval`);
+  const embedder = await createEmbedder(cacheDir, config.modelId);
+  try {
+    const records = [];
+    let dimensions = 0;
+    reportProgress(progress, `Encoding ${chunks.length} chunk${chunks.length === 1 ? "" : "s"} for dense retrieval`);
+    for (const chunk of chunks) {
+      const embedding = await embedder.embed(createDenseChunkText(chunk));
+      dimensions ||= embedding.length;
+      records.push({
+        chunkId: chunk.id,
+        documentId: chunk.documentId,
+        sourceId: chunk.sourceId,
+        title: chunk.title,
+        uri: chunk.uri,
+        headingPath: chunk.headingPath,
+        text: chunk.text,
+        embedding
+      });
+      if (records.length === 1 || records.length % 100 === 0 || records.length === chunks.length) {
+        reportProgressDetail(progress, `Encoded ${records.length}/${chunks.length} chunks for dense retrieval`);
+      }
     }
+    reportProgress(progress, "Building dense vector index");
+    const index = new VectorFieldIndex({
+      numHashTables: config.indexHashTables,
+      dimensions,
+      random: createSeededRandom(config.indexRandomSeed)
+    });
+    for (const record of records) {
+      index.insert(record.chunkId, [record.embedding]);
+    }
+    const metadata = {
+      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+      modelId: config.modelId,
+      dimensions,
+      hashTables: config.indexHashTables,
+      randomSeed: config.indexRandomSeed,
+      chunkCount: records.length,
+      indexHash: sha256(JSON.stringify(index.indexState))
+    };
+    const payload = {
+      metadata,
+      indexState: index.indexState,
+      chunks: records
+    };
+    await writeDensePayload(workspacePath, payload);
+    reportProgress(progress, `Dense vectors written for ${records.length} chunk${records.length === 1 ? "" : "s"}`);
+    return payload;
+  } finally {
+    await embedder.dispose?.();
   }
-  reportProgress(progress, "Building dense vector index");
-  const index = new VectorFieldIndex({
-    numHashTables: config.indexHashTables,
-    dimensions,
-    random: createSeededRandom(config.indexRandomSeed)
-  });
-  for (const record of records) {
-    index.insert(record.chunkId, [record.embedding]);
-  }
-  const metadata = {
-    createdAt: (/* @__PURE__ */ new Date()).toISOString(),
-    modelId: config.modelId,
-    dimensions,
-    hashTables: config.indexHashTables,
-    randomSeed: config.indexRandomSeed,
-    chunkCount: records.length,
-    indexHash: sha256(JSON.stringify(index.indexState))
-  };
-  const payload = {
-    metadata,
-    indexState: index.indexState,
-    chunks: records
-  };
-  await writeDensePayload(workspacePath, payload);
-  reportProgress(progress, `Dense vectors written for ${records.length} chunk${records.length === 1 ? "" : "s"}`);
-  return payload;
 }
 async function denseQuery({
   workspacePath,
@@ -2136,21 +2155,25 @@ async function denseQuery({
 }) {
   const payload = await readDensePayload(workspacePath);
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
-  const embed = await createEmbedder(cacheDir, config.modelId);
-  const vector = await embed(query);
-  if (payload.chunks.length <= EXACT_DENSE_RERANK_THRESHOLD) {
+  const embedder = await createEmbedder(cacheDir, config.modelId);
+  try {
+    const vector = await embedder.embed(query);
+    if (payload.chunks.length <= EXACT_DENSE_RERANK_THRESHOLD) {
+      return exactDenseQuery(payload, vector, topK);
+    }
+    const index = new VectorFieldIndex({
+      numHashTables: payload.metadata.hashTables,
+      dimensions: payload.metadata.dimensions,
+      random: createSeededRandom(payload.metadata.randomSeed)
+    }).loadState(payload.indexState);
+    const approximateHits = index.query(vector, topK);
+    if (approximateHits.length >= topK) {
+      return approximateHits;
+    }
     return exactDenseQuery(payload, vector, topK);
+  } finally {
+    await embedder.dispose?.();
   }
-  const index = new VectorFieldIndex({
-    numHashTables: payload.metadata.hashTables,
-    dimensions: payload.metadata.dimensions,
-    random: createSeededRandom(payload.metadata.randomSeed)
-  }).loadState(payload.indexState);
-  const approximateHits = index.query(vector, topK);
-  if (approximateHits.length >= topK) {
-    return approximateHits;
-  }
-  return exactDenseQuery(payload, vector, topK);
 }
 
 // src/vector/sparse.ts
@@ -2379,12 +2402,19 @@ function keywordFieldIndex() {
 function createIndexMapping(extraFields = []) {
   const lexical = new TextFieldIndex(void 0, void 0, RankingAlgorithm.BM25);
   const mapping = {
+    _source: new StoredSourceIndex(),
     text: lexical,
     title: new TextFieldIndex(void 0, void 0, RankingAlgorithm.BM25),
     uri: keywordFieldIndex(),
     sourceId: keywordFieldIndex(),
+    sourceName: keywordFieldIndex(),
     tags: keywordFieldIndex(),
-    sourceType: keywordFieldIndex()
+    sourceType: keywordFieldIndex(),
+    publicationDate: new DateFieldIndex(),
+    firstSeenAt: new DateFieldIndex(),
+    lastSeenAt: new DateFieldIndex(),
+    lastChangedAt: new DateFieldIndex(),
+    crawledAt: new DateFieldIndex()
   };
   for (const field of extraFields) {
     mapping[field] = keywordFieldIndex();
@@ -2420,8 +2450,12 @@ async function buildIndex({
   const sources = await readJsonl(path17.join(workspacePath, "sources", "sources.jsonl"));
   const metadataFields = [...new Set(chunks.flatMap((chunk) => Object.keys(chunk.metadata).map((key) => `metadata.${key}`)))];
   const index = new DocumentIndex(createIndexMapping(metadataFields));
+  const documentsById = new Map(documents.map((document) => [document.id, document]));
+  const sourcesById = new Map(sources.map((source) => [source.id, source]));
   reportProgress(progress, `Building lexical index from ${chunks.length} chunk${chunks.length === 1 ? "" : "s"}`);
   for (const chunk of chunks) {
+    const document = documentsById.get(chunk.documentId);
+    const source = sourcesById.get(chunk.sourceId);
     index.index({
       id: chunk.id,
       fields: {
@@ -2429,9 +2463,33 @@ async function buildIndex({
         title: [chunk.title],
         uri: [chunk.uri.toLowerCase()],
         sourceId: [chunk.sourceId.toLowerCase()],
+        sourceName: source ? [source.name.toLowerCase()] : [],
         tags: Array.isArray(chunk.metadata.tags) ? chunk.metadata.tags.map((tag) => String(tag).toLowerCase()) : [],
         sourceType: [String(chunk.metadata.sourceType ?? "").toLowerCase()],
+        publicationDate: document?.publicationDate ? [document.publicationDate] : [],
+        firstSeenAt: [document?.firstSeenAt ?? chunk.firstSeenAt],
+        lastSeenAt: [document?.lastSeenAt ?? chunk.lastSeenAt],
+        lastChangedAt: [document?.lastChangedAt ?? chunk.lastChangedAt],
+        crawledAt: document?.crawledAt ? [document.crawledAt] : [],
         ...flattenMetadata(chunk.metadata)
+      },
+      source: {
+        chunkId: chunk.id,
+        documentId: chunk.documentId,
+        sourceId: chunk.sourceId,
+        sourceType: document?.sourceType ?? "text",
+        sourceName: source?.name,
+        title: chunk.title,
+        uri: chunk.uri,
+        headingPath: chunk.headingPath,
+        text: chunk.text,
+        normalizedPath: document?.normalizedPath,
+        publicationDate: document?.publicationDate ?? null,
+        crawledAt: document?.crawledAt,
+        firstSeenAt: document?.firstSeenAt ?? chunk.firstSeenAt,
+        lastSeenAt: document?.lastSeenAt ?? chunk.lastSeenAt,
+        lastChangedAt: document?.lastChangedAt ?? chunk.lastChangedAt,
+        metadata: chunk.metadata
       }
     });
   }
@@ -2440,7 +2498,7 @@ async function buildIndex({
   const metadata = {
     id: `index_${createdAt.replace(/[:.]/g, "-")}`,
     createdAt,
-    querylightVersion: "0.10.0",
+    querylightVersion: "0.11.0",
     kbVersion: "0.1.0",
     documentCount: documents.length,
     chunkCount: chunks.length,
@@ -2469,7 +2527,7 @@ async function buildIndex({
 
 // src/query/search-service.ts
 import { readFile as readFile10 } from "fs/promises";
-import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
+import { reciprocalRankFusion, searchJsonDsl } from "@tryformation/querylight-ts";
 import path18 from "path";
 async function loadHydratedIndex(workspacePath) {
   let state;
@@ -2497,24 +2555,6 @@ function matchesPrefix(value, prefixes) {
   const lower = value.toLowerCase();
   return prefixes.some((prefix) => lower.startsWith(prefix));
 }
-function buildSearchQuery(query, filters) {
-  const sourceIds = normalizeFilterValues([filters.sourceId, ...filters.sourceIds ?? []].filter((value) => Boolean(value)));
-  const sourceTypes = normalizeFilterValues([filters.sourceType, ...filters.sourceTypes ?? []].filter((value) => Boolean(value)));
-  const tags = normalizeFilterValues([filters.tag, ...filters.tags ?? []].filter((value) => Boolean(value)));
-  return new BoolQuery({
-    should: [
-      new MatchQuery({ field: "title", text: query, operation: OP.AND, boost: 6 }),
-      new MatchQuery({ field: "text", text: query, operation: OP.AND, boost: 4 }),
-      new MatchQuery({ field: "text", text: query, operation: OP.OR, boost: 2 })
-    ],
-    filter: [
-      ...sourceIds.length === 1 ? [new TermQuery({ field: "sourceId", text: sourceIds[0] })] : [],
-      ...sourceTypes.length === 1 ? [new TermQuery({ field: "sourceType", text: sourceTypes[0] })] : [],
-      ...tags.length === 1 ? [new TermQuery({ field: "tags", text: tags[0] })] : [],
-      ...(filters.metadata ?? []).map(({ key, value }) => new TermQuery({ field: `metadata.${key}`, text: value.toLowerCase() }))
-    ]
-  });
-}
 function isValidDate(value) {
   return typeof value === "string" && !Number.isNaN(new Date(value).getTime());
 }
@@ -2713,6 +2753,185 @@ async function buildSnippetWithAdjacentChunks(chunk, query, {
   }
   return buildExpandedParagraphSnippet(paragraphs, currentIndex, query);
 }
+function buildSearchDslRequest({
+  query,
+  topK,
+  filters,
+  dateRanges
+}) {
+  const filterClauses = [];
+  const sourceIds = normalizeFilterValues([filters.sourceId, ...filters.sourceIds ?? []].filter((value) => Boolean(value)));
+  const sourceNames = normalizeFilterValues([filters.sourceName, ...filters.sourceNames ?? []].filter((value) => Boolean(value)));
+  const sourceTypes = normalizeFilterValues([filters.sourceType, ...filters.sourceTypes ?? []].filter((value) => Boolean(value)));
+  const uriPrefixes = normalizeFilterValues([filters.uriPrefix, ...filters.uriPrefixes ?? []].filter((value) => Boolean(value)));
+  const tags = normalizeFilterValues([filters.tag, ...filters.tags ?? []].filter((value) => Boolean(value)));
+  if (sourceIds.length > 0) {
+    filterClauses.push({ terms: { sourceId: sourceIds } });
+  }
+  if (sourceNames.length > 0) {
+    filterClauses.push({ terms: { sourceName: sourceNames } });
+  }
+  if (sourceTypes.length > 0) {
+    filterClauses.push({ terms: { sourceType: sourceTypes } });
+  }
+  if (uriPrefixes.length > 0) {
+    filterClauses.push({
+      bool: {
+        should: uriPrefixes.map((prefix) => ({ prefix: { uri: prefix } })),
+        minimum_should_match: 1
+      }
+    });
+  }
+  if (tags.length > 0) {
+    filterClauses.push({ terms: { tags } });
+  }
+  if (filters.hasPublicationDate) {
+    filterClauses.push({ exists: { field: "publicationDate" } });
+  }
+  for (const { key, value } of filters.metadata ?? []) {
+    filterClauses.push({ term: { [`metadata.${key}`]: value.toLowerCase() } });
+  }
+  for (const { field, from, to } of dateRanges) {
+    filterClauses.push({
+      range: {
+        [field]: {
+          ...from ? { gte: from } : {},
+          ...to ? { lte: to } : {}
+        }
+      }
+    });
+  }
+  return {
+    size: topK,
+    query: {
+      bool: {
+        should: [
+          { match: { title: { query, operator: "and", boost: 6 } } },
+          { match: { text: { query, operator: "and", boost: 4 } } },
+          { match: { text: { query, operator: "or", boost: 2 } } }
+        ],
+        filter: filterClauses,
+        minimum_should_match: 1
+      }
+    }
+  };
+}
+function sourceToChunkRecord(source) {
+  return {
+    id: source.chunkId,
+    documentId: source.documentId,
+    sourceId: source.sourceId,
+    title: source.title,
+    uri: source.uri,
+    headingPath: source.headingPath,
+    text: source.text,
+    contentHash: "",
+    metadata: source.metadata,
+    firstSeenAt: source.firstSeenAt,
+    lastSeenAt: source.lastSeenAt,
+    lastChangedAt: source.lastChangedAt
+  };
+}
+function sourceToDocumentRecord(source) {
+  return {
+    id: source.documentId,
+    sourceId: source.sourceId,
+    sourceType: source.sourceType,
+    title: source.title,
+    uri: source.uri,
+    sourceUri: source.uri,
+    mimeType: "text/plain",
+    normalizedPath: source.normalizedPath ?? "",
+    contentHash: "",
+    metadata: source.metadata,
+    publicationDate: source.publicationDate ?? null,
+    crawledAt: source.crawledAt,
+    firstSeenAt: source.firstSeenAt,
+    lastSeenAt: source.lastSeenAt,
+    lastChangedAt: source.lastChangedAt
+  };
+}
+async function materializeSearchHit(hit, query, config, orderedChunkCache, showChunks) {
+  const source = hit._source;
+  const chunk = sourceToChunkRecord(source);
+  const document = sourceToDocumentRecord(source);
+  const snippet = await buildSnippetWithAdjacentChunks(chunk, query, { document, config, orderedChunkCache });
+  const enrichedSource = {
+    ...source,
+    snippet
+  };
+  const result = {
+    chunkId: source.chunkId,
+    documentId: source.documentId,
+    sourceId: source.sourceId,
+    sourceType: source.sourceType,
+    score: hit._score,
+    title: chooseResultTitle(chunk),
+    uri: source.uri,
+    snippet,
+    text: showChunks ? source.text : void 0,
+    publicationDate: source.publicationDate ?? null,
+    firstSeenAt: source.firstSeenAt,
+    lastSeenAt: source.lastSeenAt,
+    lastChangedAt: source.lastChangedAt,
+    metadata: source.metadata
+  };
+  return {
+    hit: {
+      ...hit,
+      _source: enrichedSource
+    },
+    result
+  };
+}
+function createSearchResponse(retrievalMode, hits, took, aggregations) {
+  return {
+    retrievalMode,
+    took,
+    hits: {
+      total: {
+        value: hits.length,
+        relation: "eq"
+      },
+      max_score: hits.length > 0 ? Math.max(...hits.map((hit) => hit._score)) : null,
+      hits
+    },
+    aggregations
+  };
+}
+function searchResultsFromResponse(response, showChunks = false) {
+  return response.hits.hits.map((hit) => ({
+    chunkId: hit._source.chunkId,
+    documentId: hit._source.documentId,
+    sourceId: hit._source.sourceId,
+    sourceType: hit._source.sourceType,
+    score: hit._score,
+    title: chooseResultTitle(sourceToChunkRecord(hit._source)),
+    uri: hit._source.uri,
+    snippet: hit._source.snippet ?? hit.highlight?.text?.join("\n\n") ?? buildSnippet(hit._source.text, hit._source.title),
+    text: showChunks ? hit._source.text : void 0,
+    publicationDate: hit._source.publicationDate ?? null,
+    firstSeenAt: hit._source.firstSeenAt,
+    lastSeenAt: hit._source.lastSeenAt,
+    lastChangedAt: hit._source.lastChangedAt,
+    metadata: hit._source.metadata
+  }));
+}
+async function searchJsonRequest({
+  index,
+  request,
+  indexName = "querylight"
+}) {
+  return searchJsonDsl({ index, request, indexName });
+}
+async function searchJsonIndex({
+  workspacePath,
+  request,
+  indexName = "querylight"
+}) {
+  const index = await loadHydratedIndex(workspacePath);
+  return searchJsonRequest({ index, request, indexName });
+}
 function normalizeDisplayTitle(title) {
   return title.replace(/\s*\|\s*Querylight TS Demo\s*$/i, "").replace(/\s+/g, " ").trim();
 }
@@ -2850,6 +3069,7 @@ async function searchIndex({
   retrievalMode,
   showChunks = false
 }) {
+  const startedAt = Date.now();
   const config = await loadConfig(workspacePath);
   const mode = retrievalMode ?? config.retrieval.defaultMode;
   const candidateLimit = Math.max(topK * 5, 50);
@@ -2906,12 +3126,48 @@ async function searchIndex({
         };
       })
     );
-    return { retrievalMode: "lexical", results: latestResults.filter((result) => result != null) };
+    const hits2 = latestResults.filter((result) => result != null).map((result) => {
+      const chunk = chunks.get(result.chunkId);
+      const document = documents.get(result.documentId);
+      const source = sources.get(result.sourceId);
+      return {
+        _index: "querylight",
+        _id: result.chunkId,
+        _score: result.score,
+        _source: {
+          chunkId: result.chunkId,
+          documentId: result.documentId,
+          sourceId: result.sourceId,
+          sourceType: result.sourceType,
+          sourceName: source?.name,
+          title: chunk.title,
+          uri: result.uri,
+          headingPath: chunk.headingPath,
+          text: chunk.text,
+          snippet: result.snippet,
+          normalizedPath: document.normalizedPath,
+          publicationDate: result.publicationDate ?? null,
+          crawledAt: document.crawledAt,
+          firstSeenAt: result.firstSeenAt,
+          lastSeenAt: result.lastSeenAt,
+          lastChangedAt: result.lastChangedAt,
+          metadata: result.metadata
+        }
+      };
+    });
+    return createSearchResponse("lexical", hits2, Date.now() - startedAt);
   }
   const lexicalHits = async () => {
-    const index = await loadHydratedIndex(workspacePath);
-    const all = await index.searchRequest({ query: buildSearchQuery(normalizedQuery, { sourceId, sourceIds, sourceType, sourceTypes, tag, tags, metadata }), limit: candidateLimit });
-    return all.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit);
+    const response = await searchJsonIndex({
+      workspacePath,
+      request: buildSearchDslRequest({
+        query: normalizedQuery,
+        topK: candidateLimit,
+        filters: { sourceId, sourceIds, sourceName, sourceNames, sourceType, sourceTypes, uriPrefix, uriPrefixes, hasPublicationDate, tag, tags, metadata },
+        dateRanges
+      })
+    });
+    return response.hits.hits;
   };
   const denseHits = async () => {
     if (!await fileExists(denseVectorPath(workspacePath))) {
@@ -2925,15 +3181,18 @@ async function searchIndex({
     }
     return sparseQuery({ workspacePath, config: config.retrieval.sparse, query: normalizedQuery, topK: candidateLimit }).then((hits2) => hits2.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit));
   };
+  let lexicalResponseHits = [];
   let hits;
   if (mode === "lexical") {
-    hits = await lexicalHits();
+    lexicalResponseHits = await lexicalHits();
+    hits = lexicalResponseHits.map((hit) => [hit._id, hit._score]);
   } else if (mode === "dense") {
     hits = await denseHits();
   } else if (mode === "sparse") {
     hits = await sparseHits();
   } else {
-    const rankings = [await lexicalHits()];
+    lexicalResponseHits = await lexicalHits();
+    const rankings = [lexicalResponseHits.map((hit) => [hit._id, hit._score])];
     if (await fileExists(denseVectorPath(workspacePath))) {
       rankings.push(await denseQuery({ workspacePath, config: config.retrieval.dense, query: normalizedQuery, topK: candidateLimit }).then((dense) => dense.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit)));
     }
@@ -2942,38 +3201,242 @@ async function searchIndex({
     }
     hits = reciprocalRankFusion(rankings, { rankConstant: 20, weights: rankings.map((_, index) => index === 0 ? 3 : 1) }).slice(0, candidateLimit);
   }
-  const rawResults = await Promise.all(hits.map(async ([chunkId, score]) => {
+  const baseHits = mode === "lexical" ? lexicalResponseHits : hits.flatMap(([chunkId, score]) => {
     const chunk = chunks.get(chunkId);
     if (!chunk) {
-      return null;
+      return [];
     }
+    const document = documents.get(chunk.documentId);
+    const source = sources.get(chunk.sourceId);
+    return [{
+      _index: "querylight",
+      _id: chunkId,
+      _score: score,
+      _source: {
+        chunkId,
+        documentId: chunk.documentId,
+        sourceId: chunk.sourceId,
+        sourceType: document?.sourceType ?? "text",
+        sourceName: source?.name,
+        title: chunk.title,
+        uri: chunk.uri,
+        headingPath: chunk.headingPath,
+        text: chunk.text,
+        normalizedPath: document?.normalizedPath,
+        publicationDate: document?.publicationDate ?? null,
+        crawledAt: document?.crawledAt,
+        firstSeenAt: document?.firstSeenAt ?? chunk.firstSeenAt,
+        lastSeenAt: document?.lastSeenAt ?? chunk.lastSeenAt,
+        lastChangedAt: document?.lastChangedAt ?? chunk.lastChangedAt,
+        metadata: chunk.metadata
+      }
+    }];
+  });
+  const materialized = await Promise.all(baseHits.map((hit) => materializeSearchHit(hit, normalizedQuery, config, orderedChunkCache, showChunks)));
+  if (showChunks) {
+    const topHits = materialized.sort((left, right) => right.result.score - left.result.score).slice(0, topK).map(({ hit, result }) => ({ ...hit, _score: result.score }));
+    return createSearchResponse(mode, topHits, Date.now() - startedAt);
+  }
+  const reranked = rerankResultsByDocument(materialized.map(({ result }) => result), topK);
+  const byChunkId = new Map(materialized.map(({ hit }) => [hit._id, hit]));
+  const finalHits = reranked.map((result) => {
+    const hit = byChunkId.get(result.chunkId);
+    return hit ? { ...hit, _score: result.score, _source: { ...hit._source, snippet: result.snippet } } : null;
+  }).filter((hit) => hit != null);
+  return createSearchResponse(mode, finalHits, Date.now() - startedAt);
+}
+
+// src/server/search-api.ts
+import { createServer } from "http";
+import { readdir, stat as stat4 } from "fs/promises";
+import path19 from "path";
+async function pathIsDirectory(candidatePath) {
+  try {
+    return (await stat4(candidatePath)).isDirectory();
+  } catch {
+    return false;
+  }
+}
+async function discoverKnowledgeBases(workspacePath) {
+  try {
+    const singleWorkspace = await assertWorkspaceExists(workspacePath);
+    const config = await loadConfig(singleWorkspace);
+    const index = await loadHydratedIndex(singleWorkspace);
     return {
-      chunkId,
-      documentId: chunk.documentId,
-      sourceId: chunk.sourceId,
-      sourceType: documents.get(chunk.documentId)?.sourceType ?? "text",
-      score,
-      title: chooseResultTitle(chunk),
-      uri: chunk.uri,
-      snippet: await buildSnippetWithAdjacentChunks(chunk, normalizedQuery, {
-        document: documents.get(chunk.documentId),
-        config,
-        orderedChunkCache
-      }),
-      text: showChunks ? chunk.text : void 0,
-      publicationDate: documents.get(chunk.documentId)?.publicationDate ?? null,
-      firstSeenAt: documents.get(chunk.documentId)?.firstSeenAt ?? chunk.firstSeenAt,
-      lastSeenAt: documents.get(chunk.documentId)?.lastSeenAt ?? chunk.lastSeenAt,
-      lastChangedAt: documents.get(chunk.documentId)?.lastChangedAt ?? chunk.lastChangedAt,
-      metadata: chunk.metadata
+      mode: "single",
+      knowledgeBases: [{
+        name: config.index.name,
+        workspacePath: singleWorkspace,
+        configuredIndexName: config.index.name,
+        index
+      }]
     };
-  }));
-  const results = rawResults.filter((result) => result != null);
-  return { retrievalMode: mode, results: rerankResultsByDocument(results, topK) };
+  } catch (error) {
+    if (!(error instanceof CliError) || error.code !== "WORKSPACE_ERROR") {
+      throw error;
+    }
+  }
+  const resolvedRoot = path19.resolve(workspacePath);
+  if (!await pathIsDirectory(resolvedRoot)) {
+    throw new CliError(`workspace path does not exist: ${resolvedRoot}`, "WORKSPACE_ERROR", 3 /* WorkspaceError */);
+  }
+  const entries = await readdir(resolvedRoot, { withFileTypes: true });
+  const knowledgeBases = (await Promise.all(entries.filter((entry) => entry.isDirectory()).map(async (entry) => {
+    const candidateWorkspace = path19.join(resolvedRoot, entry.name, ".kb");
+    try {
+      const workspace = await assertWorkspaceExists(candidateWorkspace);
+      const config = await loadConfig(workspace);
+      const index = await loadHydratedIndex(workspace);
+      return {
+        name: entry.name,
+        workspacePath: workspace,
+        configuredIndexName: config.index.name,
+        index
+      };
+    } catch (error) {
+      if (error instanceof CliError && error.code === "WORKSPACE_ERROR") {
+        return null;
+      }
+      throw error;
+    }
+  }))).filter((knowledgeBase) => knowledgeBase != null);
+  if (knowledgeBases.length === 0) {
+    throw new CliError(
+      `no knowledge bases found at ${resolvedRoot}; use a .kb workspace or a directory of named subdirectories that each contain .kb`,
+      "WORKSPACE_ERROR",
+      3 /* WorkspaceError */
+    );
+  }
+  return { mode: "multi", knowledgeBases };
+}
+function sendJson(response, statusCode, payload) {
+  response.statusCode = statusCode;
+  response.setHeader("content-type", "application/json; charset=utf-8");
+  response.end(JSON.stringify(payload));
+}
+function sendError(response, statusCode, type, reason) {
+  sendJson(response, statusCode, {
+    error: {
+      type,
+      reason
+    },
+    status: statusCode
+  });
+}
+async function readRequestBody(request) {
+  const chunks = [];
+  for await (const chunk of request) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks).toString("utf8");
+}
+function parseSearchRequest(raw) {
+  const normalized = raw.trim();
+  if (normalized.length === 0) {
+    return {};
+  }
+  try {
+    const parsed = JSON.parse(normalized);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new Error("expected a JSON object");
+    }
+    return parsed;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new CliError(`invalid JSON request: ${message}`, "INVALID_ARGUMENT", 2 /* InvalidArguments */);
+  }
+}
+function routeForKnowledgeBase(mode, knowledgeBase) {
+  return mode === "single" ? "/_search" : `/${knowledgeBase.name}/_search`;
+}
+function resolveKnowledgeBaseForPath(pathname, mode, knowledgeBases) {
+  const segments = pathname.split("/").filter(Boolean);
+  if (mode === "single") {
+    const knowledgeBase = [...knowledgeBases.values()][0];
+    if (!knowledgeBase) {
+      return null;
+    }
+    if (segments.length === 1 && segments[0] === "_search") {
+      return knowledgeBase;
+    }
+    if (segments.length === 2 && segments[1] === "_search" && segments[0] === knowledgeBase.configuredIndexName) {
+      return knowledgeBase;
+    }
+    return null;
+  }
+  if (segments.length === 2 && segments[1] === "_search") {
+    return knowledgeBases.get(segments[0]) ?? null;
+  }
+  return null;
+}
+async function handleSearchRequest(request, response, pathname, mode, knowledgeBases) {
+  if (request.method !== "GET" && request.method !== "POST") {
+    response.setHeader("allow", "GET, POST");
+    sendError(response, 405, "method_not_allowed", `unsupported method for ${pathname}`);
+    return;
+  }
+  const knowledgeBase = resolveKnowledgeBaseForPath(pathname, mode, knowledgeBases);
+  if (!knowledgeBase) {
+    sendError(response, 404, "resource_not_found_exception", `unknown search route: ${pathname}`);
+    return;
+  }
+  try {
+    const requestBody = parseSearchRequest(await readRequestBody(request));
+    const indexName = mode === "multi" ? knowledgeBase.name : knowledgeBase.configuredIndexName;
+    const result = await searchJsonRequest({
+      index: knowledgeBase.index,
+      request: requestBody,
+      indexName
+    });
+    sendJson(response, 200, result);
+  } catch (error) {
+    if (error instanceof CliError && error.code === "INVALID_ARGUMENT") {
+      sendError(response, 400, "parse_exception", error.message);
+      return;
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    sendError(response, 500, "search_phase_execution_exception", message);
+  }
+}
+async function startSearchApiServer({
+  workspacePath,
+  host = "127.0.0.1",
+  port = 3e3
+}) {
+  const { mode, knowledgeBases } = await discoverKnowledgeBases(workspacePath);
+  const byName = new Map(knowledgeBases.map((knowledgeBase) => [knowledgeBase.name, knowledgeBase]));
+  const server = createServer(async (request, response) => {
+    const url2 = new URL(request.url ?? "/", `http://${request.headers.host ?? `${host}:${port}`}`);
+    await handleSearchRequest(request, response, url2.pathname, mode, byName);
+  });
+  await new Promise((resolve2, reject) => {
+    server.once("error", reject);
+    server.listen(port, host, () => {
+      server.off("error", reject);
+      resolve2();
+    });
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new CliError("server failed to bind to a TCP address", "SERVER_ERROR", 1 /* GeneralError */);
+  }
+  const url = `http://${host}:${address.port}`;
+  return {
+    mode,
+    url,
+    knowledgeBases: knowledgeBases.map((knowledgeBase) => ({
+      name: knowledgeBase.name,
+      workspacePath: knowledgeBase.workspacePath,
+      route: routeForKnowledgeBase(mode, knowledgeBase)
+    })),
+    close: async () => new Promise((resolve2, reject) => {
+      server.close((error) => error ? reject(error) : resolve2());
+    })
+  };
 }
 
 // src/query/related-service.ts
-import path19 from "path";
+import path20 from "path";
 function cosineSimilarity2(left, right) {
   let dot = 0;
   let leftNorm = 0;
@@ -3049,7 +3512,7 @@ async function findRelatedDocuments({
   if (!await fileExists(denseVectorPath(workspacePath))) {
     throw new CliError("dense vector index is not built; run `qli models pull --dense` and `qli rebuild`", "DENSE_INDEX_MISSING", 7 /* QueryError */);
   }
-  const documents = await readJsonl(path19.join(workspacePath, "documents", "documents.jsonl"));
+  const documents = await readJsonl(path20.join(workspacePath, "documents", "documents.jsonl"));
   const selected = resolveDocumentSelector(documents, document);
   const densePayload = await readDensePayload(workspacePath);
   const vectors = buildDocumentVectors(documents, densePayload.chunks, densePayload.metadata.dimensions);
@@ -3086,9 +3549,10 @@ async function createContext({
   retrievalMode
 }) {
   const search = await searchIndex({ workspacePath, query, topK, showChunks: true, retrievalMode });
+  const results = searchResultsFromResponse(search, true);
   const sources = [];
   let total = 0;
-  for (const result of search.results) {
+  for (const result of results) {
     const text = result.text ?? "";
     if (total + text.length > maxChars && sources.length > 0) {
       break;
@@ -3121,7 +3585,7 @@ async function createContext({
 }
 
 // src/report/diff-service.ts
-import path20 from "path";
+import path21 from "path";
 function chooseBaselineRun(runs, since) {
   if (since === "last-run") {
     return runs.at(-1);
@@ -3137,7 +3601,7 @@ async function diffWorkspace({
   documentId,
   since
 }) {
-  const current = await readJsonl(path20.join(workspacePath, "documents", "documents.jsonl"));
+  const current = await readJsonl(path21.join(workspacePath, "documents", "documents.jsonl"));
   const baseline = chooseBaselineRun(await listRuns(workspacePath), since);
   const previous = new Map((baseline?.documentsSnapshot ?? []).map((document) => [document.id, document]));
   const changedDocuments = current.filter((document) => (!sourceId || document.sourceId === sourceId) && (!documentId || document.id === documentId)).filter((document) => {
@@ -3193,10 +3657,15 @@ export {
   ingestSources,
   listSources,
   loadConfig,
+  loadHydratedIndex,
   removeSource,
   renderChangeReport,
   reprocessDocuments,
   searchIndex,
+  searchJsonIndex,
+  searchJsonRequest,
+  searchResultsFromResponse,
+  startSearchApiServer,
   updateSource,
   writeDefaultConfig
 };