@tryformation/querylight-cli 0.2.1 → 0.2.3

package/dist/core/constants.d.ts

@@ -1,5 +1,5 @@
 export declare const PACKAGE_NAME = "@tryformation/querylight-cli";
-export declare const PACKAGE_VERSION = "0.2.1";
+export declare const PACKAGE_VERSION = "0.2.3";
 export declare const DEFAULT_WORKSPACE = ".kb";
 export declare const DEFAULT_SHARED_MODEL_CACHE_DIR = "~/.qli/models/huggingface";
 export declare const LEGACY_WORKSPACE_MODEL_CACHE_DIR = ".kb/models/huggingface";

package/dist/index/querylight-indexer.d.ts

@@ -1,7 +1,7 @@
-import { TextFieldIndex } from "@tryformation/querylight-ts";
+import { type FieldIndex } from "@tryformation/querylight-ts";
 import { type ProgressHandler } from "../core/progress.js";
 import type { IndexMetadata } from "../types/models.js";
-export declare function createIndexMapping(extraFields?: string[]): Record<string, TextFieldIndex>;
+export declare function createIndexMapping(extraFields?: string[]): Record<string, FieldIndex>;
 export declare function buildIndex({ workspacePath, denseOverride, sparseOverride, buildAvailableModels, progress }: {
     workspacePath: string;
     denseOverride?: boolean;

package/dist/index.js

@@ -1782,7 +1782,7 @@ async function chunkDocuments({
 }
 
 // src/index/querylight-indexer.ts
-import { Analyzer, DocumentIndex, KeywordTokenizer, LowerCaseTextFilter, RankingAlgorithm, TextFieldIndex } from "@tryformation/querylight-ts";
+import { Analyzer, DateFieldIndex, DocumentIndex, KeywordTokenizer, LowerCaseTextFilter, RankingAlgorithm, StoredSourceIndex, TextFieldIndex } from "@tryformation/querylight-ts";
 import path17 from "path";
 
 // src/vector/dense.ts
@@ -1795,6 +1795,8 @@ import os from "os";
 import path12 from "path";
 import { fileURLToPath } from "url";
 import { execFile, execFileSync } from "child_process";
+import { mkdtemp, rm as rm2, writeFile as writeFile6 } from "fs/promises";
+var sparseExecFileSync = execFileSync;
 function resolveQliHomeDir() {
   return path12.resolve(process.env.QLI_HOME ?? path12.join(os.homedir(), ".qli"));
 }
@@ -1850,29 +1852,36 @@ async function runSparsePython({
 }) {
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
   const scriptPath = await sparseScriptPath(importMetaUrl);
-  return execFileSync(
-    "uv",
-    [
-      "run",
-      "--with",
-      "torch",
-      "--with",
-      "transformers",
-      "--with",
-      "huggingface_hub",
-      "python",
-      scriptPath
-    ],
-    {
-      encoding: "utf8",
-      maxBuffer: 1024 * 1024 * 1024,
-      input: JSON.stringify(payload),
-      env: {
-        ...process.env,
-        HF_HOME: cacheDir
+  const payloadDir = await mkdtemp(path12.join(os.tmpdir(), "qli-sparse-"));
+  const payloadPath = path12.join(payloadDir, "payload.json");
+  await writeFile6(payloadPath, JSON.stringify(payload), "utf8");
+  try {
+    return sparseExecFileSync(
+      "uv",
+      [
+        "run",
+        "--with",
+        "torch",
+        "--with",
+        "transformers",
+        "--with",
+        "huggingface_hub",
+        "python",
+        scriptPath,
+        payloadPath
+      ],
+      {
+        encoding: "utf8",
+        maxBuffer: 1024 * 1024 * 1024,
+        env: {
+          ...process.env,
+          HF_HOME: cacheDir
+        }
       }
-    }
-  );
+    );
+  } finally {
+    await rm2(payloadDir, { recursive: true, force: true });
+  }
 }
 async function getDenseTransformersRuntime(cacheDir) {
   const transformers = await import("@huggingface/transformers");
@@ -1885,18 +1894,18 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir6, rm as rm2, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm3, writeFile as writeFile8 } from "fs/promises";
 import path13 from "path";
 
 // src/core/gzip-json.ts
-import { readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
+import { readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
 import { promisify } from "util";
 import { gunzip, gzip } from "zlib";
 var gzipAsync = promisify(gzip);
 var gunzipAsync = promisify(gunzip);
 async function writeGzipJson(filePath, value) {
   const payload = JSON.stringify(value, null, 2);
-  await writeFile6(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+  await writeFile7(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
 }
 async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
   if (await fileExists(gzipPath)) {
@@ -1956,8 +1965,8 @@ async function writeDensePayload(workspacePath, payload) {
   await writeGzipJson(denseVectorPath(workspacePath), payload);
   await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
   await Promise.all([
-    rm2(legacyDenseVectorPath(workspacePath), { force: true }),
-    rm2(legacyDenseMetaPath(workspacePath), { force: true })
+    rm3(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm3(legacyDenseMetaPath(workspacePath), { force: true })
   ]);
 }
 async function readDensePayload(workspacePath) {
@@ -1968,8 +1977,8 @@ async function writeSparsePayload(workspacePath, payload) {
   await writeGzipJson(sparseVectorPath(workspacePath), payload);
   await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
   await Promise.all([
-    rm2(legacySparseVectorPath(workspacePath), { force: true }),
-    rm2(legacySparseMetaPath(workspacePath), { force: true })
+    rm3(legacySparseVectorPath(workspacePath), { force: true }),
+    rm3(legacySparseMetaPath(workspacePath), { force: true })
   ]);
 }
 async function readSparsePayload(workspacePath) {
@@ -2309,7 +2318,7 @@ async function buildVectorArtifacts({
 }
 
 // src/index/index-store.ts
-import { mkdir as mkdir9, rm as rm3 } from "fs/promises";
+import { mkdir as mkdir9, rm as rm4 } from "fs/promises";
 import path16 from "path";
 function versionedIndexPath(workspacePath, stamp) {
   return path16.join(workspacePath, "indexes", `${stamp}.json.gz`);
@@ -2351,10 +2360,10 @@ async function writeIndexArtifacts({
   await writeGzipJson(latestIndexArtifactPath, indexState);
   await writeGzipJson(latestMetadataArtifactPath, metadata);
   await Promise.all([
-    rm3(legacyLatestIndexPath(workspacePath), { force: true }),
-    rm3(legacyLatestMetaPath(workspacePath), { force: true }),
-    rm3(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
-    rm3(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+    rm4(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm4(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm4(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm4(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
   ]);
   return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }
@@ -2370,12 +2379,19 @@ function keywordFieldIndex() {
 function createIndexMapping(extraFields = []) {
   const lexical = new TextFieldIndex(void 0, void 0, RankingAlgorithm.BM25);
   const mapping = {
+    _source: new StoredSourceIndex(),
     text: lexical,
     title: new TextFieldIndex(void 0, void 0, RankingAlgorithm.BM25),
     uri: keywordFieldIndex(),
     sourceId: keywordFieldIndex(),
+    sourceName: keywordFieldIndex(),
     tags: keywordFieldIndex(),
-    sourceType: keywordFieldIndex()
+    sourceType: keywordFieldIndex(),
+    publicationDate: new DateFieldIndex(),
+    firstSeenAt: new DateFieldIndex(),
+    lastSeenAt: new DateFieldIndex(),
+    lastChangedAt: new DateFieldIndex(),
+    crawledAt: new DateFieldIndex()
   };
   for (const field of extraFields) {
     mapping[field] = keywordFieldIndex();
@@ -2411,8 +2427,12 @@ async function buildIndex({
   const sources = await readJsonl(path17.join(workspacePath, "sources", "sources.jsonl"));
   const metadataFields = [...new Set(chunks.flatMap((chunk) => Object.keys(chunk.metadata).map((key) => `metadata.${key}`)))];
   const index = new DocumentIndex(createIndexMapping(metadataFields));
+  const documentsById = new Map(documents.map((document) => [document.id, document]));
+  const sourcesById = new Map(sources.map((source) => [source.id, source]));
   reportProgress(progress, `Building lexical index from ${chunks.length} chunk${chunks.length === 1 ? "" : "s"}`);
   for (const chunk of chunks) {
+    const document = documentsById.get(chunk.documentId);
+    const source = sourcesById.get(chunk.sourceId);
     index.index({
       id: chunk.id,
       fields: {
@@ -2420,9 +2440,33 @@ async function buildIndex({
         title: [chunk.title],
         uri: [chunk.uri.toLowerCase()],
         sourceId: [chunk.sourceId.toLowerCase()],
+        sourceName: source ? [source.name.toLowerCase()] : [],
         tags: Array.isArray(chunk.metadata.tags) ? chunk.metadata.tags.map((tag) => String(tag).toLowerCase()) : [],
         sourceType: [String(chunk.metadata.sourceType ?? "").toLowerCase()],
+        publicationDate: document?.publicationDate ? [document.publicationDate] : [],
+        firstSeenAt: [document?.firstSeenAt ?? chunk.firstSeenAt],
+        lastSeenAt: [document?.lastSeenAt ?? chunk.lastSeenAt],
+        lastChangedAt: [document?.lastChangedAt ?? chunk.lastChangedAt],
+        crawledAt: document?.crawledAt ? [document.crawledAt] : [],
         ...flattenMetadata(chunk.metadata)
+      },
+      source: {
+        chunkId: chunk.id,
+        documentId: chunk.documentId,
+        sourceId: chunk.sourceId,
+        sourceType: document?.sourceType ?? "text",
+        sourceName: source?.name,
+        title: chunk.title,
+        uri: chunk.uri,
+        headingPath: chunk.headingPath,
+        text: chunk.text,
+        normalizedPath: document?.normalizedPath,
+        publicationDate: document?.publicationDate ?? null,
+        crawledAt: document?.crawledAt,
+        firstSeenAt: document?.firstSeenAt ?? chunk.firstSeenAt,
+        lastSeenAt: document?.lastSeenAt ?? chunk.lastSeenAt,
+        lastChangedAt: document?.lastChangedAt ?? chunk.lastChangedAt,
+        metadata: chunk.metadata
       }
     });
   }
@@ -2431,7 +2475,7 @@ async function buildIndex({
   const metadata = {
     id: `index_${createdAt.replace(/[:.]/g, "-")}`,
     createdAt,
-    querylightVersion: "0.10.0",
+    querylightVersion: "0.11.0",
     kbVersion: "0.1.0",
     documentCount: documents.length,
     chunkCount: chunks.length,
@@ -2460,7 +2504,7 @@ async function buildIndex({
 
 // src/query/search-service.ts
 import { readFile as readFile10 } from "fs/promises";
-import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
+import { reciprocalRankFusion, searchJsonDsl } from "@tryformation/querylight-ts";
 import path18 from "path";
 async function loadHydratedIndex(workspacePath) {
   let state;
@@ -2488,24 +2532,6 @@ function matchesPrefix(value, prefixes) {
   const lower = value.toLowerCase();
   return prefixes.some((prefix) => lower.startsWith(prefix));
 }
-function buildSearchQuery(query, filters) {
-  const sourceIds = normalizeFilterValues([filters.sourceId, ...filters.sourceIds ?? []].filter((value) => Boolean(value)));
-  const sourceTypes = normalizeFilterValues([filters.sourceType, ...filters.sourceTypes ?? []].filter((value) => Boolean(value)));
-  const tags = normalizeFilterValues([filters.tag, ...filters.tags ?? []].filter((value) => Boolean(value)));
-  return new BoolQuery({
-    should: [
-      new MatchQuery({ field: "title", text: query, operation: OP.AND, boost: 6 }),
-      new MatchQuery({ field: "text", text: query, operation: OP.AND, boost: 4 }),
-      new MatchQuery({ field: "text", text: query, operation: OP.OR, boost: 2 })
-    ],
-    filter: [
-      ...sourceIds.length === 1 ? [new TermQuery({ field: "sourceId", text: sourceIds[0] })] : [],
-      ...sourceTypes.length === 1 ? [new TermQuery({ field: "sourceType", text: sourceTypes[0] })] : [],
-      ...tags.length === 1 ? [new TermQuery({ field: "tags", text: tags[0] })] : [],
-      ...(filters.metadata ?? []).map(({ key, value }) => new TermQuery({ field: `metadata.${key}`, text: value.toLowerCase() }))
-    ]
-  });
-}
 function isValidDate(value) {
   return typeof value === "string" && !Number.isNaN(new Date(value).getTime());
 }
@@ -2704,6 +2730,178 @@ async function buildSnippetWithAdjacentChunks(chunk, query, {
   }
   return buildExpandedParagraphSnippet(paragraphs, currentIndex, query);
 }
+function buildSearchDslRequest({
+  query,
+  topK,
+  filters,
+  dateRanges
+}) {
+  const filterClauses = [];
+  const sourceIds = normalizeFilterValues([filters.sourceId, ...filters.sourceIds ?? []].filter((value) => Boolean(value)));
+  const sourceNames = normalizeFilterValues([filters.sourceName, ...filters.sourceNames ?? []].filter((value) => Boolean(value)));
+  const sourceTypes = normalizeFilterValues([filters.sourceType, ...filters.sourceTypes ?? []].filter((value) => Boolean(value)));
+  const uriPrefixes = normalizeFilterValues([filters.uriPrefix, ...filters.uriPrefixes ?? []].filter((value) => Boolean(value)));
+  const tags = normalizeFilterValues([filters.tag, ...filters.tags ?? []].filter((value) => Boolean(value)));
+  if (sourceIds.length > 0) {
+    filterClauses.push({ terms: { sourceId: sourceIds } });
+  }
+  if (sourceNames.length > 0) {
+    filterClauses.push({ terms: { sourceName: sourceNames } });
+  }
+  if (sourceTypes.length > 0) {
+    filterClauses.push({ terms: { sourceType: sourceTypes } });
+  }
+  if (uriPrefixes.length > 0) {
+    filterClauses.push({
+      bool: {
+        should: uriPrefixes.map((prefix) => ({ prefix: { uri: prefix } })),
+        minimum_should_match: 1
+      }
+    });
+  }
+  if (tags.length > 0) {
+    filterClauses.push({ terms: { tags } });
+  }
+  if (filters.hasPublicationDate) {
+    filterClauses.push({ exists: { field: "publicationDate" } });
+  }
+  for (const { key, value } of filters.metadata ?? []) {
+    filterClauses.push({ term: { [`metadata.${key}`]: value.toLowerCase() } });
+  }
+  for (const { field, from, to } of dateRanges) {
+    filterClauses.push({
+      range: {
+        [field]: {
+          ...from ? { gte: from } : {},
+          ...to ? { lte: to } : {}
+        }
+      }
+    });
+  }
+  return {
+    size: topK,
+    query: {
+      bool: {
+        should: [
+          { match: { title: { query, operator: "and", boost: 6 } } },
+          { match: { text: { query, operator: "and", boost: 4 } } },
+          { match: { text: { query, operator: "or", boost: 2 } } }
+        ],
+        filter: filterClauses,
+        minimum_should_match: 1
+      }
+    }
+  };
+}
+function sourceToChunkRecord(source) {
+  return {
+    id: source.chunkId,
+    documentId: source.documentId,
+    sourceId: source.sourceId,
+    title: source.title,
+    uri: source.uri,
+    headingPath: source.headingPath,
+    text: source.text,
+    contentHash: "",
+    metadata: source.metadata,
+    firstSeenAt: source.firstSeenAt,
+    lastSeenAt: source.lastSeenAt,
+    lastChangedAt: source.lastChangedAt
+  };
+}
+function sourceToDocumentRecord(source) {
+  return {
+    id: source.documentId,
+    sourceId: source.sourceId,
+    sourceType: source.sourceType,
+    title: source.title,
+    uri: source.uri,
+    sourceUri: source.uri,
+    mimeType: "text/plain",
+    normalizedPath: source.normalizedPath ?? "",
+    contentHash: "",
+    metadata: source.metadata,
+    publicationDate: source.publicationDate ?? null,
+    crawledAt: source.crawledAt,
+    firstSeenAt: source.firstSeenAt,
+    lastSeenAt: source.lastSeenAt,
+    lastChangedAt: source.lastChangedAt
+  };
+}
+async function materializeSearchHit(hit, query, config, orderedChunkCache, showChunks) {
+  const source = hit._source;
+  const chunk = sourceToChunkRecord(source);
+  const document = sourceToDocumentRecord(source);
+  const snippet = await buildSnippetWithAdjacentChunks(chunk, query, { document, config, orderedChunkCache });
+  const enrichedSource = {
+    ...source,
+    snippet
+  };
+  const result = {
+    chunkId: source.chunkId,
+    documentId: source.documentId,
+    sourceId: source.sourceId,
+    sourceType: source.sourceType,
+    score: hit._score,
+    title: chooseResultTitle(chunk),
+    uri: source.uri,
+    snippet,
+    text: showChunks ? source.text : void 0,
+    publicationDate: source.publicationDate ?? null,
+    firstSeenAt: source.firstSeenAt,
+    lastSeenAt: source.lastSeenAt,
+    lastChangedAt: source.lastChangedAt,
+    metadata: source.metadata
+  };
+  return {
+    hit: {
+      ...hit,
+      _source: enrichedSource
+    },
+    result
+  };
+}
+function createSearchResponse(retrievalMode, hits, took, aggregations) {
+  return {
+    retrievalMode,
+    took,
+    hits: {
+      total: {
+        value: hits.length,
+        relation: "eq"
+      },
+      max_score: hits.length > 0 ? Math.max(...hits.map((hit) => hit._score)) : null,
+      hits
+    },
+    aggregations
+  };
+}
+function searchResultsFromResponse(response, showChunks = false) {
+  return response.hits.hits.map((hit) => ({
+    chunkId: hit._source.chunkId,
+    documentId: hit._source.documentId,
+    sourceId: hit._source.sourceId,
+    sourceType: hit._source.sourceType,
+    score: hit._score,
+    title: chooseResultTitle(sourceToChunkRecord(hit._source)),
+    uri: hit._source.uri,
+    snippet: hit._source.snippet ?? hit.highlight?.text?.join("\n\n") ?? buildSnippet(hit._source.text, hit._source.title),
+    text: showChunks ? hit._source.text : void 0,
+    publicationDate: hit._source.publicationDate ?? null,
+    firstSeenAt: hit._source.firstSeenAt,
+    lastSeenAt: hit._source.lastSeenAt,
+    lastChangedAt: hit._source.lastChangedAt,
+    metadata: hit._source.metadata
+  }));
+}
+async function searchJsonIndex({
+  workspacePath,
+  request,
+  indexName = "querylight"
+}) {
+  const index = await loadHydratedIndex(workspacePath);
+  return searchJsonDsl({ index, request, indexName });
+}
 function normalizeDisplayTitle(title) {
   return title.replace(/\s*\|\s*Querylight TS Demo\s*$/i, "").replace(/\s+/g, " ").trim();
 }
@@ -2841,6 +3039,7 @@ async function searchIndex({
   retrievalMode,
   showChunks = false
 }) {
+  const startedAt = Date.now();
   const config = await loadConfig(workspacePath);
   const mode = retrievalMode ?? config.retrieval.defaultMode;
   const candidateLimit = Math.max(topK * 5, 50);
@@ -2897,12 +3096,48 @@ async function searchIndex({
         };
       })
     );
-    return { retrievalMode: "lexical", results: latestResults.filter((result) => result != null) };
+    const hits2 = latestResults.filter((result) => result != null).map((result) => {
+      const chunk = chunks.get(result.chunkId);
+      const document = documents.get(result.documentId);
+      const source = sources.get(result.sourceId);
+      return {
+        _index: "querylight",
+        _id: result.chunkId,
+        _score: result.score,
+        _source: {
+          chunkId: result.chunkId,
+          documentId: result.documentId,
+          sourceId: result.sourceId,
+          sourceType: result.sourceType,
+          sourceName: source?.name,
+          title: chunk.title,
+          uri: result.uri,
+          headingPath: chunk.headingPath,
+          text: chunk.text,
+          snippet: result.snippet,
+          normalizedPath: document.normalizedPath,
+          publicationDate: result.publicationDate ?? null,
+          crawledAt: document.crawledAt,
+          firstSeenAt: result.firstSeenAt,
+          lastSeenAt: result.lastSeenAt,
+          lastChangedAt: result.lastChangedAt,
+          metadata: result.metadata
+        }
+      };
+    });
+    return createSearchResponse("lexical", hits2, Date.now() - startedAt);
   }
   const lexicalHits = async () => {
-    const index = await loadHydratedIndex(workspacePath);
-    const all = await index.searchRequest({ query: buildSearchQuery(normalizedQuery, { sourceId, sourceIds, sourceType, sourceTypes, tag, tags, metadata }), limit: candidateLimit });
-    return all.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit);
+    const response = await searchJsonIndex({
+      workspacePath,
+      request: buildSearchDslRequest({
+        query: normalizedQuery,
+        topK: candidateLimit,
+        filters: { sourceId, sourceIds, sourceName, sourceNames, sourceType, sourceTypes, uriPrefix, uriPrefixes, hasPublicationDate, tag, tags, metadata },
+        dateRanges
+      })
+    });
+    return response.hits.hits;
   };
   const denseHits = async () => {
     if (!await fileExists(denseVectorPath(workspacePath))) {
@@ -2916,15 +3151,18 @@ async function searchIndex({
     }
     return sparseQuery({ workspacePath, config: config.retrieval.sparse, query: normalizedQuery, topK: candidateLimit }).then((hits2) => hits2.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit));
   };
+  let lexicalResponseHits = [];
   let hits;
   if (mode === "lexical") {
-    hits = await lexicalHits();
+    lexicalResponseHits = await lexicalHits();
+    hits = lexicalResponseHits.map((hit) => [hit._id, hit._score]);
   } else if (mode === "dense") {
     hits = await denseHits();
   } else if (mode === "sparse") {
     hits = await sparseHits();
   } else {
-    const rankings = [await lexicalHits()];
+    lexicalResponseHits = await lexicalHits();
+    const rankings = [lexicalResponseHits.map((hit) => [hit._id, hit._score])];
     if (await fileExists(denseVectorPath(workspacePath))) {
       rankings.push(await denseQuery({ workspacePath, config: config.retrieval.dense, query: normalizedQuery, topK: candidateLimit }).then((dense) => dense.filter(([chunkId]) => filterIds.includes(chunkId)).slice(0, candidateLimit)));
     }
@@ -2933,34 +3171,49 @@ async function searchIndex({
     }
     hits = reciprocalRankFusion(rankings, { rankConstant: 20, weights: rankings.map((_, index) => index === 0 ? 3 : 1) }).slice(0, candidateLimit);
   }
-  const rawResults = await Promise.all(hits.map(async ([chunkId, score]) => {
+  const baseHits = mode === "lexical" ? lexicalResponseHits : hits.flatMap(([chunkId, score]) => {
     const chunk = chunks.get(chunkId);
     if (!chunk) {
-      return null;
+      return [];
     }
-    return {
-      chunkId,
-      documentId: chunk.documentId,
-      sourceId: chunk.sourceId,
-      sourceType: documents.get(chunk.documentId)?.sourceType ?? "text",
-      score,
-      title: chooseResultTitle(chunk),
-      uri: chunk.uri,
-      snippet: await buildSnippetWithAdjacentChunks(chunk, normalizedQuery, {
-        document: documents.get(chunk.documentId),
-        config,
-        orderedChunkCache
-      }),
-      text: showChunks ? chunk.text : void 0,
-      publicationDate: documents.get(chunk.documentId)?.publicationDate ?? null,
-      firstSeenAt: documents.get(chunk.documentId)?.firstSeenAt ?? chunk.firstSeenAt,
-      lastSeenAt: documents.get(chunk.documentId)?.lastSeenAt ?? chunk.lastSeenAt,
-      lastChangedAt: documents.get(chunk.documentId)?.lastChangedAt ?? chunk.lastChangedAt,
-      metadata: chunk.metadata
-    };
-  }));
-  const results = rawResults.filter((result) => result != null);
-  return { retrievalMode: mode, results: rerankResultsByDocument(results, topK) };
+    const document = documents.get(chunk.documentId);
+    const source = sources.get(chunk.sourceId);
+    return [{
+      _index: "querylight",
+      _id: chunkId,
+      _score: score,
+      _source: {
+        chunkId,
+        documentId: chunk.documentId,
+        sourceId: chunk.sourceId,
+        sourceType: document?.sourceType ?? "text",
+        sourceName: source?.name,
+        title: chunk.title,
+        uri: chunk.uri,
+        headingPath: chunk.headingPath,
+        text: chunk.text,
+        normalizedPath: document?.normalizedPath,
+        publicationDate: document?.publicationDate ?? null,
+        crawledAt: document?.crawledAt,
+        firstSeenAt: document?.firstSeenAt ?? chunk.firstSeenAt,
+        lastSeenAt: document?.lastSeenAt ?? chunk.lastSeenAt,
+        lastChangedAt: document?.lastChangedAt ?? chunk.lastChangedAt,
+        metadata: chunk.metadata
+      }
+    }];
+  });
+  const materialized = await Promise.all(baseHits.map((hit) => materializeSearchHit(hit, normalizedQuery, config, orderedChunkCache, showChunks)));
+  if (showChunks) {
+    const topHits = materialized.sort((left, right) => right.result.score - left.result.score).slice(0, topK).map(({ hit, result }) => ({ ...hit, _score: result.score }));
+    return createSearchResponse(mode, topHits, Date.now() - startedAt);
+  }
+  const reranked = rerankResultsByDocument(materialized.map(({ result }) => result), topK);
+  const byChunkId = new Map(materialized.map(({ hit }) => [hit._id, hit]));
+  const finalHits = reranked.map((result) => {
+    const hit = byChunkId.get(result.chunkId);
+    return hit ? { ...hit, _score: result.score, _source: { ...hit._source, snippet: result.snippet } } : null;
+  }).filter((hit) => hit != null);
+  return createSearchResponse(mode, finalHits, Date.now() - startedAt);
 }
 
 // src/query/related-service.ts
@@ -3077,9 +3330,10 @@ async function createContext({
   retrievalMode
 }) {
   const search = await searchIndex({ workspacePath, query, topK, showChunks: true, retrievalMode });
+  const results = searchResultsFromResponse(search, true);
   const sources = [];
   let total = 0;
-  for (const result of search.results) {
+  for (const result of results) {
     const text = result.text ?? "";
     if (total + text.length > maxChars && sources.length > 0) {
       break;
@@ -3188,6 +3442,8 @@ export {
   renderChangeReport,
   reprocessDocuments,
   searchIndex,
+  searchJsonIndex,
+  searchResultsFromResponse,
   updateSource,
   writeDefaultConfig
 };

package/dist/query/search-service.d.ts

@@ -1,10 +1,17 @@
-import type { RetrievalMode, SearchResponseData } from "../types/models.js";
+import { type JsonDslRequest, type JsonDslResponse } from "@tryformation/querylight-ts";
+import type { RetrievalMode, SearchResponseData, SearchResult } from "../types/models.js";
 type SearchDateField = "publicationDate" | "firstSeenAt" | "lastSeenAt" | "lastChangedAt" | "crawledAt";
 type SearchDateRange = {
     field: SearchDateField;
     from?: string;
     to?: string;
 };
+export declare function searchResultsFromResponse(response: SearchResponseData, showChunks?: boolean): SearchResult[];
+export declare function searchJsonIndex({ workspacePath, request, indexName }: {
+    workspacePath: string;
+    request: JsonDslRequest;
+    indexName?: string;
+}): Promise<JsonDslResponse>;
 export declare function searchIndex({ workspacePath, query, topK, sourceId, sourceIds, sourceName, sourceNames, sourceType, sourceTypes, uriPrefix, uriPrefixes, hasPublicationDate, tag, tags, metadata, dateRanges, retrievalMode, showChunks }: {
     workspacePath: string;
     query: string;