@tryformation/querylight-cli 0.2.0 → 0.2.2

package/README.md

@@ -208,8 +208,8 @@ The default workspace is `.kb/`.
   raw/
   normalized/
   indexes/
-    latest.json
-    latest.meta.json
+    latest.json.gz
+    latest.meta.json.gz
   runs/
   logs/
 ```

package/dist/cli/main.js

@@ -16,7 +16,7 @@ import path from "path";
 import YAML from "yaml";
 
 // src/core/constants.ts
-var PACKAGE_VERSION = "0.2.0";
+var PACKAGE_VERSION = "0.2.1";
 var DEFAULT_WORKSPACE = ".kb";
 var DEFAULT_SHARED_MODEL_CACHE_DIR = "~/.qli/models/huggingface";
 var LEGACY_WORKSPACE_MODEL_CACHE_DIR = ".kb/models/huggingface";
@@ -387,6 +387,7 @@ import os from "os";
 import path6 from "path";
 import { fileURLToPath } from "url";
 import { execFile, execFileSync } from "child_process";
+import { mkdtemp, rm, writeFile as writeFile3 } from "fs/promises";
 
 // src/core/files.ts
 import { stat as stat2 } from "fs/promises";
@@ -400,6 +401,7 @@ async function fileExists(filePath) {
 }
 
 // src/vector/runtime.ts
+var sparseExecFileSync = execFileSync;
 function resolveQliHomeDir() {
   return path6.resolve(process.env.QLI_HOME ?? path6.join(os.homedir(), ".qli"));
 }
@@ -455,29 +457,36 @@ async function runSparsePython({
 }) {
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
   const scriptPath = await sparseScriptPath(importMetaUrl);
-  return execFileSync(
-    "uv",
-    [
-      "run",
-      "--with",
-      "torch",
-      "--with",
-      "transformers",
-      "--with",
-      "huggingface_hub",
-      "python",
-      scriptPath
-    ],
-    {
-      encoding: "utf8",
-      maxBuffer: 1024 * 1024 * 1024,
-      input: JSON.stringify(payload),
-      env: {
-        ...process.env,
-        HF_HOME: cacheDir
+  const payloadDir = await mkdtemp(path6.join(os.tmpdir(), "qli-sparse-"));
+  const payloadPath = path6.join(payloadDir, "payload.json");
+  await writeFile3(payloadPath, JSON.stringify(payload), "utf8");
+  try {
+    return sparseExecFileSync(
+      "uv",
+      [
+        "run",
+        "--with",
+        "torch",
+        "--with",
+        "transformers",
+        "--with",
+        "huggingface_hub",
+        "python",
+        scriptPath,
+        payloadPath
+      ],
+      {
+        encoding: "utf8",
+        maxBuffer: 1024 * 1024 * 1024,
+        env: {
+          ...process.env,
+          HF_HOME: cacheDir
+        }
       }
-    }
-  );
+    );
+  } finally {
+    await rm(payloadDir, { recursive: true, force: true });
+  }
 }
 async function getDenseTransformersRuntime(cacheDir) {
   const transformers = await import("@huggingface/transformers");
@@ -490,8 +499,40 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir3, readFile as readFile4, writeFile as writeFile3 } from "fs/promises";
+import { mkdir as mkdir3, rm as rm2, writeFile as writeFile5 } from "fs/promises";
 import path7 from "path";
+
+// src/core/gzip-json.ts
+import { readFile as readFile4, writeFile as writeFile4 } from "fs/promises";
+import { promisify } from "util";
+import { gunzip, gzip } from "zlib";
+var gzipAsync = promisify(gzip);
+var gunzipAsync = promisify(gunzip);
+async function writeGzipJson(filePath, value) {
+  const payload = JSON.stringify(value, null, 2);
+  await writeFile4(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+}
+async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
+  if (await fileExists(gzipPath)) {
+    const payload = await readFile4(gzipPath);
+    return JSON.parse((await gunzipAsync(payload)).toString("utf8"));
+  }
+  if (legacyPath && await fileExists(legacyPath)) {
+    return JSON.parse(await readFile4(legacyPath, "utf8"));
+  }
+  return JSON.parse(await readFile4(gzipPath, "utf8"));
+}
+async function resolveExistingGzipOrFilePath(gzipPath, legacyPath) {
+  if (await fileExists(gzipPath)) {
+    return gzipPath;
+  }
+  if (legacyPath && await fileExists(legacyPath)) {
+    return legacyPath;
+  }
+  return gzipPath;
+}
+
+// src/vector/store.ts
 function vectorsDir(workspacePath) {
   return path7.join(workspacePath, "vectors");
 }
@@ -499,15 +540,27 @@ function sharedModelStateDir() {
   return path7.join(resolveQliHomeDir(), "models", "status");
 }
 function denseVectorPath(workspacePath) {
-  return path7.join(vectorsDir(workspacePath), "dense.latest.json");
+  return path7.join(vectorsDir(workspacePath), "dense.latest.json.gz");
 }
 function denseMetaPath(workspacePath) {
-  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json.gz");
 }
 function sparseVectorPath(workspacePath) {
-  return path7.join(vectorsDir(workspacePath), "sparse.latest.json");
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.json.gz");
 }
 function sparseMetaPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.meta.json.gz");
+}
+function legacyDenseVectorPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "dense.latest.json");
+}
+function legacyDenseMetaPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+}
+function legacySparseVectorPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.json");
+}
+function legacySparseMetaPath(workspacePath) {
   return path7.join(vectorsDir(workspacePath), "sparse.latest.meta.json");
 }
 function pullMarkerPath(type, workspacePath, modelId, cacheDir) {
@@ -523,29 +576,37 @@ function sparsePullMarker(workspacePath, modelId, cacheDir) {
 }
 async function writeDensePayload(workspacePath, payload) {
   await mkdir3(vectorsDir(workspacePath), { recursive: true });
-  await writeFile3(denseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile3(denseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(denseVectorPath(workspacePath), payload);
+  await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm2(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm2(legacyDenseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readDensePayload(workspacePath) {
-  return JSON.parse(await readFile4(denseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(denseVectorPath(workspacePath), legacyDenseVectorPath(workspacePath));
 }
 async function writeSparsePayload(workspacePath, payload) {
   await mkdir3(vectorsDir(workspacePath), { recursive: true });
-  await writeFile3(sparseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile3(sparseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(sparseVectorPath(workspacePath), payload);
+  await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm2(legacySparseVectorPath(workspacePath), { force: true }),
+    rm2(legacySparseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readSparsePayload(workspacePath) {
-  return JSON.parse(await readFile4(sparseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(sparseVectorPath(workspacePath), legacySparseVectorPath(workspacePath));
 }
 async function writeDensePullMarker(workspacePath, model, value) {
   const markerPath = densePullMarker(workspacePath, model.modelId, model.cacheDir);
   await mkdir3(path7.dirname(markerPath), { recursive: true });
-  await writeFile3(markerPath, JSON.stringify(value, null, 2), "utf8");
+  await writeFile5(markerPath, JSON.stringify(value, null, 2), "utf8");
 }
 async function writeSparsePullMarker(workspacePath, model, value) {
   const markerPath = sparsePullMarker(workspacePath, model.modelId, model.cacheDir);
   await mkdir3(path7.dirname(markerPath), { recursive: true });
-  await writeFile3(markerPath, JSON.stringify(value, null, 2), "utf8");
+  await writeFile5(markerPath, JSON.stringify(value, null, 2), "utf8");
 }
 async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
   const denseCacheDir = resolveCacheDir(workspacePath, dense.cacheDir);
@@ -556,7 +617,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       modelId: dense.modelId,
       cacheDir: denseCacheDir,
       available: await fileExists(densePullMarker(workspacePath, dense.modelId, dense.cacheDir)),
-      artifactExists: await fileExists(denseVectorPath(workspacePath))
+      artifactExists: await fileExists(denseVectorPath(workspacePath)) || await fileExists(legacyDenseVectorPath(workspacePath))
     },
     sparse: {
       configured: sparse.enabled,
@@ -564,7 +625,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       cacheDir: sparseCacheDir,
       uvAvailable,
       available: await fileExists(sparsePullMarker(workspacePath, sparse.modelId, sparse.cacheDir)),
-      artifactExists: await fileExists(sparseVectorPath(workspacePath))
+      artifactExists: await fileExists(sparseVectorPath(workspacePath)) || await fileExists(legacySparseVectorPath(workspacePath))
     }
   };
 }
@@ -963,31 +1024,63 @@ async function getModelStatus(workspacePath, config) {
 }
 
 // src/index/index-store.ts
-import { readFile as readFile5, writeFile as writeFile4 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm3 } from "fs/promises";
 import path10 from "path";
+function versionedIndexPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.json.gz`);
+}
+function versionedLegacyIndexPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.json`);
+}
+function versionedMetaPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.meta.json.gz`);
+}
+function versionedLegacyMetaPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.meta.json`);
+}
+function latestIndexPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.json.gz");
+}
+function legacyLatestIndexPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.json");
+}
+function latestMetaPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.meta.json.gz");
+}
+function legacyLatestMetaPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.meta.json");
+}
 async function writeIndexArtifacts({
   workspacePath,
   indexState,
   metadata
 }) {
   const stamp = metadata.createdAt.replace(/[:.]/g, "-");
-  const indexPath = path10.join(workspacePath, "indexes", `${stamp}.json`);
-  const metaPath = path10.join(workspacePath, "indexes", `${stamp}.meta.json`);
-  const latestIndexPath = path10.join(workspacePath, "indexes", "latest.json");
-  const latestMetaPath = path10.join(workspacePath, "indexes", "latest.meta.json");
-  const indexPayload = JSON.stringify(indexState, null, 2);
-  const metaPayload = JSON.stringify(metadata, null, 2);
-  await writeFile4(indexPath, indexPayload, "utf8");
-  await writeFile4(metaPath, metaPayload, "utf8");
-  await writeFile4(latestIndexPath, indexPayload, "utf8");
-  await writeFile4(latestMetaPath, metaPayload, "utf8");
-  return { indexPath: latestIndexPath, metadataPath: latestMetaPath };
+  const indexPath = versionedIndexPath(workspacePath, stamp);
+  const metaPath = versionedMetaPath(workspacePath, stamp);
+  const latestIndexArtifactPath = latestIndexPath(workspacePath);
+  const latestMetadataArtifactPath = latestMetaPath(workspacePath);
+  await mkdir6(path10.join(workspacePath, "indexes"), { recursive: true });
+  await writeGzipJson(indexPath, indexState);
+  await writeGzipJson(metaPath, metadata);
+  await writeGzipJson(latestIndexArtifactPath, indexState);
+  await writeGzipJson(latestMetadataArtifactPath, metadata);
+  await Promise.all([
+    rm3(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm3(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm3(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm3(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+  ]);
+  return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }
 async function readLatestIndexState(workspacePath) {
-  return JSON.parse(await readFile5(path10.join(workspacePath, "indexes", "latest.json"), "utf8"));
+  return readJsonFromGzipOrFile(latestIndexPath(workspacePath), legacyLatestIndexPath(workspacePath));
 }
 async function readLatestIndexMetadata(workspacePath) {
-  return JSON.parse(await readFile5(path10.join(workspacePath, "indexes", "latest.meta.json"), "utf8"));
+  return readJsonFromGzipOrFile(latestMetaPath(workspacePath), legacyLatestMetaPath(workspacePath));
+}
+async function resolveLatestIndexArtifactPath(workspacePath) {
+  return resolveExistingGzipOrFilePath(latestIndexPath(workspacePath), legacyLatestIndexPath(workspacePath));
 }
 
 // src/index/querylight-indexer.ts
@@ -1178,7 +1271,7 @@ async function removeSource(workspacePath, sourceId) {
 }
 
 // src/ingest/document-utils.ts
-import { mkdir as mkdir6, rm, writeFile as writeFile5 } from "fs/promises";
+import { mkdir as mkdir7, rm as rm4, writeFile as writeFile6 } from "fs/promises";
 import path14 from "path";
 
 // src/normalize/normalize-markdown.ts
@@ -1231,8 +1324,8 @@ async function writeNormalizedDocument({
   normalizedPath,
   markdown
 }) {
-  await mkdir6(path14.dirname(normalizedPath), { recursive: true });
-  await writeFile5(
+  await mkdir7(path14.dirname(normalizedPath), { recursive: true });
+  await writeFile6(
     normalizedPath,
     withFrontmatter(
       {
@@ -1254,8 +1347,8 @@ async function writeNormalizedDocument({
 }
 async function deleteDocumentArtifacts(document) {
   await Promise.all([
-    document.rawPath ? rm(document.rawPath, { force: true }) : Promise.resolve(),
-    rm(document.normalizedPath, { force: true })
+    document.rawPath ? rm4(document.rawPath, { force: true }) : Promise.resolve(),
+    rm4(document.normalizedPath, { force: true })
   ]);
 }
 
@@ -1279,7 +1372,7 @@ async function listDirectoryFiles(source) {
 
 // src/ingest/adapters/file-adapter.ts
 import { basename, extname, resolve } from "path";
-import { mkdir as mkdir7, readFile as readFile9, stat as stat3, writeFile as writeFile6 } from "fs/promises";
+import { mkdir as mkdir8, readFile as readFile8, stat as stat3, writeFile as writeFile7 } from "fs/promises";
 
 // src/ingest/extractors/docx-extractor.ts
 import mammoth from "mammoth";
@@ -1453,16 +1546,16 @@ function extractPublicationDateFromHtml(html) {
 }
 
 // src/ingest/extractors/markdown-extractor.ts
-import { readFile as readFile6 } from "fs/promises";
+import { readFile as readFile5 } from "fs/promises";
 async function extractMarkdown(filePath) {
-  return readFile6(filePath, "utf8");
+  return readFile5(filePath, "utf8");
 }
 
 // src/ingest/extractors/pdf-extractor.ts
-import { readFile as readFile7 } from "fs/promises";
+import { readFile as readFile6 } from "fs/promises";
 import { PDFParse } from "pdf-parse";
 async function extractPdf(filePath) {
-  const buffer = await readFile7(filePath);
+  const buffer = await readFile6(filePath);
   const parser = new PDFParse({ data: buffer });
   try {
     const parsed = await parser.getText();
@@ -1473,9 +1566,9 @@ async function extractPdf(filePath) {
 }
 
 // src/ingest/extractors/text-extractor.ts
-import { readFile as readFile8 } from "fs/promises";
+import { readFile as readFile7 } from "fs/promises";
 async function extractText(filePath) {
-  return readFile8(filePath, "utf8");
+  return readFile7(filePath, "utf8");
 }
 
 // src/ingest/adapters/file-adapter.ts
@@ -1510,7 +1603,7 @@ async function extractFileContent(filePath, mimeType) {
 ${text}`, raw: text };
   }
   if (mimeType === "text/html") {
-    const raw = await readFile9(filePath, "utf8");
+    const raw = await readFile8(filePath, "utf8");
     const extracted = extractHtmlToMarkdown(raw);
     return { title: extracted.title, markdown: `# ${extracted.title}
 
@@ -1566,10 +1659,10 @@ async function ingestFile({
   const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
   const indexedAt = now;
   const crawledAt = now;
-  await mkdir7(resolve(workspacePath, "normalized"), { recursive: true });
-  await mkdir7(resolve(workspacePath, "raw", source.id), { recursive: true });
+  await mkdir8(resolve(workspacePath, "normalized"), { recursive: true });
+  await mkdir8(resolve(workspacePath, "raw", source.id), { recursive: true });
   if (extracted.raw) {
-    await writeFile6(rawPath, extracted.raw, "utf8");
+    await writeFile7(rawPath, extracted.raw, "utf8");
   }
   await writeNormalizedDocument({
     documentId,
@@ -1632,7 +1725,7 @@ ${content}`;
   const now = (/* @__PURE__ */ new Date()).toISOString();
   const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
   const indexedAt = now;
-  await mkdir7(resolve(workspacePath, "normalized"), { recursive: true });
+  await mkdir8(resolve(workspacePath, "normalized"), { recursive: true });
   await writeNormalizedDocument({
     documentId,
     sourceId: source.id,
@@ -1676,7 +1769,7 @@ async function reprocessStoredDocument(document, source) {
   if (!document.rawPath) {
     return null;
   }
-  const raw = await readFile9(document.rawPath, "utf8");
+  const raw = await readFile8(document.rawPath, "utf8");
   const fallbackTitle = document.title || basename(document.uri);
   const extracted = await extractRawContent(raw, document.mimeType, fallbackTitle);
   const contentHash = sha256(extracted.markdown);
@@ -1793,7 +1886,7 @@ async function parseRssFeedDocument(xml, source) {
 }
 
 // src/ingest/adapters/url-adapter.ts
-import { mkdir as mkdir8, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir9, readFile as readFile9, writeFile as writeFile8 } from "fs/promises";
 import path16 from "path";
 
 // src/core/urls.ts
@@ -1845,8 +1938,8 @@ ${extracted.markdown}`;
   const indexedAt = now;
   const crawledAt = now;
   const resolvedPublicationDate = choosePublicationDate(publicationDate, extractPublicationDateFromHtml(body), previous?.publicationDate);
-  await mkdir8(path16.resolve(workspacePath, "raw", source.id), { recursive: true });
-  await writeFile7(rawPath, body, "utf8");
+  await mkdir9(path16.resolve(workspacePath, "raw", source.id), { recursive: true });
+  await writeFile8(rawPath, body, "utf8");
   await writeNormalizedDocument({
     documentId,
     sourceId: source.id,
@@ -1966,7 +2059,7 @@ async function reprocessRemoteDocument(document, source) {
   if (!document.rawPath || !await fileExists(document.rawPath)) {
     return null;
   }
-  const raw = await readFile10(document.rawPath, "utf8");
+  const raw = await readFile9(document.rawPath, "utf8");
   const extracted = extractHtmlToMarkdown(raw);
   const markdown = `# ${extracted.title}
 
@@ -2684,7 +2777,7 @@ async function discoverWebsiteFeed(websiteUrl, userAgent) {
 }
 
 // src/query/search-service.ts
-import { readFile as readFile11 } from "fs/promises";
+import { readFile as readFile10 } from "fs/promises";
 import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
 import path18 from "path";
 async function loadHydratedIndex(workspacePath) {
@@ -2914,7 +3007,7 @@ async function buildSnippetWithAdjacentChunks(chunk, query, {
     if (!await fileExists(document.normalizedPath)) {
       return buildSnippet(chunk.text, query);
     }
-    const raw = await readFile11(document.normalizedPath, "utf8");
+    const raw = await readFile10(document.normalizedPath, "utf8");
     orderedChunks = buildChunksForDocument(document, raw, config);
     orderedChunkCache.set(document.id, orderedChunks);
   }
@@ -4210,7 +4303,7 @@ Examples:
     try {
       const meta = await readLatestIndexMetadata(workspace);
       latestIndex = meta.createdAt;
-      indexSize = (await stat4(`${workspace}/indexes/latest.json`)).size;
+      indexSize = (await stat4(await resolveLatestIndexArtifactPath(workspace))).size;
     } catch {
       latestIndex = void 0;
     }

package/dist/core/constants.d.ts

@@ -1,5 +1,5 @@
 export declare const PACKAGE_NAME = "@tryformation/querylight-cli";
-export declare const PACKAGE_VERSION = "0.2.0";
+export declare const PACKAGE_VERSION = "0.2.1";
 export declare const DEFAULT_WORKSPACE = ".kb";
 export declare const DEFAULT_SHARED_MODEL_CACHE_DIR = "~/.qli/models/huggingface";
 export declare const LEGACY_WORKSPACE_MODEL_CACHE_DIR = ".kb/models/huggingface";

package/dist/core/gzip-json.d.ts

@@ -0,0 +1,3 @@
+export declare function writeGzipJson(filePath: string, value: unknown): Promise<void>;
+export declare function readJsonFromGzipOrFile<T>(gzipPath: string, legacyPath?: string): Promise<T>;
+export declare function resolveExistingGzipOrFilePath(gzipPath: string, legacyPath?: string): Promise<string>;

package/dist/index/index-store.d.ts

@@ -1,4 +1,6 @@
 import type { IndexMetadata } from "../types/models.js";
+export declare function latestIndexPath(workspacePath: string): string;
+export declare function latestMetaPath(workspacePath: string): string;
 export declare function writeIndexArtifacts({ workspacePath, indexState, metadata }: {
     workspacePath: string;
     indexState: object;
@@ -9,3 +11,4 @@ export declare function writeIndexArtifacts({ workspacePath, indexState, metadat
 }>;
 export declare function readLatestIndexState(workspacePath: string): Promise<object>;
 export declare function readLatestIndexMetadata(workspacePath: string): Promise<IndexMetadata>;
+export declare function resolveLatestIndexArtifactPath(workspacePath: string): Promise<string>;

package/dist/index.js

@@ -1795,6 +1795,8 @@ import os from "os";
 import path12 from "path";
 import { fileURLToPath } from "url";
 import { execFile, execFileSync } from "child_process";
+import { mkdtemp, rm as rm2, writeFile as writeFile6 } from "fs/promises";
+var sparseExecFileSync = execFileSync;
 function resolveQliHomeDir() {
   return path12.resolve(process.env.QLI_HOME ?? path12.join(os.homedir(), ".qli"));
 }
@@ -1850,29 +1852,36 @@ async function runSparsePython({
 }) {
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
   const scriptPath = await sparseScriptPath(importMetaUrl);
-  return execFileSync(
-    "uv",
-    [
-      "run",
-      "--with",
-      "torch",
-      "--with",
-      "transformers",
-      "--with",
-      "huggingface_hub",
-      "python",
-      scriptPath
-    ],
-    {
-      encoding: "utf8",
-      maxBuffer: 1024 * 1024 * 1024,
-      input: JSON.stringify(payload),
-      env: {
-        ...process.env,
-        HF_HOME: cacheDir
+  const payloadDir = await mkdtemp(path12.join(os.tmpdir(), "qli-sparse-"));
+  const payloadPath = path12.join(payloadDir, "payload.json");
+  await writeFile6(payloadPath, JSON.stringify(payload), "utf8");
+  try {
+    return sparseExecFileSync(
+      "uv",
+      [
+        "run",
+        "--with",
+        "torch",
+        "--with",
+        "transformers",
+        "--with",
+        "huggingface_hub",
+        "python",
+        scriptPath,
+        payloadPath
+      ],
+      {
+        encoding: "utf8",
+        maxBuffer: 1024 * 1024 * 1024,
+        env: {
+          ...process.env,
+          HF_HOME: cacheDir
+        }
       }
-    }
-  );
+    );
+  } finally {
+    await rm2(payloadDir, { recursive: true, force: true });
+  }
 }
 async function getDenseTransformersRuntime(cacheDir) {
   const transformers = await import("@huggingface/transformers");
@@ -1885,8 +1894,31 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir6, readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm3, writeFile as writeFile8 } from "fs/promises";
 import path13 from "path";
+
+// src/core/gzip-json.ts
+import { readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
+import { promisify } from "util";
+import { gunzip, gzip } from "zlib";
+var gzipAsync = promisify(gzip);
+var gunzipAsync = promisify(gunzip);
+async function writeGzipJson(filePath, value) {
+  const payload = JSON.stringify(value, null, 2);
+  await writeFile7(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+}
+async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
+  if (await fileExists(gzipPath)) {
+    const payload = await readFile9(gzipPath);
+    return JSON.parse((await gunzipAsync(payload)).toString("utf8"));
+  }
+  if (legacyPath && await fileExists(legacyPath)) {
+    return JSON.parse(await readFile9(legacyPath, "utf8"));
+  }
+  return JSON.parse(await readFile9(gzipPath, "utf8"));
+}
+
+// src/vector/store.ts
 function vectorsDir(workspacePath) {
   return path13.join(workspacePath, "vectors");
 }
@@ -1894,15 +1926,27 @@ function sharedModelStateDir() {
   return path13.join(resolveQliHomeDir(), "models", "status");
 }
 function denseVectorPath(workspacePath) {
-  return path13.join(vectorsDir(workspacePath), "dense.latest.json");
+  return path13.join(vectorsDir(workspacePath), "dense.latest.json.gz");
 }
 function denseMetaPath(workspacePath) {
-  return path13.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+  return path13.join(vectorsDir(workspacePath), "dense.latest.meta.json.gz");
 }
 function sparseVectorPath(workspacePath) {
-  return path13.join(vectorsDir(workspacePath), "sparse.latest.json");
+  return path13.join(vectorsDir(workspacePath), "sparse.latest.json.gz");
 }
 function sparseMetaPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "sparse.latest.meta.json.gz");
+}
+function legacyDenseVectorPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "dense.latest.json");
+}
+function legacyDenseMetaPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+}
+function legacySparseVectorPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "sparse.latest.json");
+}
+function legacySparseMetaPath(workspacePath) {
   return path13.join(vectorsDir(workspacePath), "sparse.latest.meta.json");
 }
 function pullMarkerPath(type, workspacePath, modelId, cacheDir) {
@@ -1918,19 +1962,27 @@ function sparsePullMarker(workspacePath, modelId, cacheDir) {
 }
 async function writeDensePayload(workspacePath, payload) {
   await mkdir6(vectorsDir(workspacePath), { recursive: true });
-  await writeFile6(denseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile6(denseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(denseVectorPath(workspacePath), payload);
+  await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm3(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm3(legacyDenseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readDensePayload(workspacePath) {
-  return JSON.parse(await readFile9(denseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(denseVectorPath(workspacePath), legacyDenseVectorPath(workspacePath));
 }
 async function writeSparsePayload(workspacePath, payload) {
   await mkdir6(vectorsDir(workspacePath), { recursive: true });
-  await writeFile6(sparseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile6(sparseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(sparseVectorPath(workspacePath), payload);
+  await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm3(legacySparseVectorPath(workspacePath), { force: true }),
+    rm3(legacySparseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readSparsePayload(workspacePath) {
-  return JSON.parse(await readFile9(sparseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(sparseVectorPath(workspacePath), legacySparseVectorPath(workspacePath));
 }
 async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
   const denseCacheDir = resolveCacheDir(workspacePath, dense.cacheDir);
@@ -1941,7 +1993,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       modelId: dense.modelId,
       cacheDir: denseCacheDir,
       available: await fileExists(densePullMarker(workspacePath, dense.modelId, dense.cacheDir)),
-      artifactExists: await fileExists(denseVectorPath(workspacePath))
+      artifactExists: await fileExists(denseVectorPath(workspacePath)) || await fileExists(legacyDenseVectorPath(workspacePath))
     },
     sparse: {
       configured: sparse.enabled,
@@ -1949,7 +2001,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       cacheDir: sparseCacheDir,
       uvAvailable,
       available: await fileExists(sparsePullMarker(workspacePath, sparse.modelId, sparse.cacheDir)),
-      artifactExists: await fileExists(sparseVectorPath(workspacePath))
+      artifactExists: await fileExists(sparseVectorPath(workspacePath)) || await fileExists(legacySparseVectorPath(workspacePath))
     }
   };
 }
@@ -2266,28 +2318,57 @@ async function buildVectorArtifacts({
 }
 
 // src/index/index-store.ts
-import { readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir9, rm as rm4 } from "fs/promises";
 import path16 from "path";
+function versionedIndexPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.json.gz`);
+}
+function versionedLegacyIndexPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.json`);
+}
+function versionedMetaPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.meta.json.gz`);
+}
+function versionedLegacyMetaPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.meta.json`);
+}
+function latestIndexPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.json.gz");
+}
+function legacyLatestIndexPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.json");
+}
+function latestMetaPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.meta.json.gz");
+}
+function legacyLatestMetaPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.meta.json");
+}
 async function writeIndexArtifacts({
   workspacePath,
   indexState,
   metadata
 }) {
   const stamp = metadata.createdAt.replace(/[:.]/g, "-");
-  const indexPath = path16.join(workspacePath, "indexes", `${stamp}.json`);
-  const metaPath = path16.join(workspacePath, "indexes", `${stamp}.meta.json`);
-  const latestIndexPath = path16.join(workspacePath, "indexes", "latest.json");
-  const latestMetaPath = path16.join(workspacePath, "indexes", "latest.meta.json");
-  const indexPayload = JSON.stringify(indexState, null, 2);
-  const metaPayload = JSON.stringify(metadata, null, 2);
-  await writeFile7(indexPath, indexPayload, "utf8");
-  await writeFile7(metaPath, metaPayload, "utf8");
-  await writeFile7(latestIndexPath, indexPayload, "utf8");
-  await writeFile7(latestMetaPath, metaPayload, "utf8");
-  return { indexPath: latestIndexPath, metadataPath: latestMetaPath };
+  const indexPath = versionedIndexPath(workspacePath, stamp);
+  const metaPath = versionedMetaPath(workspacePath, stamp);
+  const latestIndexArtifactPath = latestIndexPath(workspacePath);
+  const latestMetadataArtifactPath = latestMetaPath(workspacePath);
+  await mkdir9(path16.join(workspacePath, "indexes"), { recursive: true });
+  await writeGzipJson(indexPath, indexState);
+  await writeGzipJson(metaPath, metadata);
+  await writeGzipJson(latestIndexArtifactPath, indexState);
+  await writeGzipJson(latestMetadataArtifactPath, metadata);
+  await Promise.all([
+    rm4(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm4(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm4(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm4(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+  ]);
+  return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }
 async function readLatestIndexState(workspacePath) {
-  return JSON.parse(await readFile10(path16.join(workspacePath, "indexes", "latest.json"), "utf8"));
+  return readJsonFromGzipOrFile(latestIndexPath(workspacePath), legacyLatestIndexPath(workspacePath));
 }
 
 // src/index/querylight-indexer.ts
@@ -2387,7 +2468,7 @@ async function buildIndex({
 }
 
 // src/query/search-service.ts
-import { readFile as readFile11 } from "fs/promises";
+import { readFile as readFile10 } from "fs/promises";
 import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
 import path18 from "path";
 async function loadHydratedIndex(workspacePath) {
@@ -2617,7 +2698,7 @@ async function buildSnippetWithAdjacentChunks(chunk, query, {
     if (!await fileExists(document.normalizedPath)) {
       return buildSnippet(chunk.text, query);
     }
-    const raw = await readFile11(document.normalizedPath, "utf8");
+    const raw = await readFile10(document.normalizedPath, "utf8");
     orderedChunks = buildChunksForDocument(document, raw, config);
     orderedChunkCache.set(document.id, orderedChunks);
   }

package/dist/vector/runtime.d.ts

@@ -1,4 +1,11 @@
 import type { SparseVectorModelConfig } from "../types/models.js";
+type SparseExecOptions = {
+    encoding: BufferEncoding;
+    maxBuffer: number;
+    env: NodeJS.ProcessEnv;
+};
+type SparseExecFileSync = (file: string, args: string[], options: SparseExecOptions) => string;
+export declare function setSparseExecFileSyncForTests(fn: SparseExecFileSync | null): void;
 export declare function resolveQliHomeDir(): string;
 export declare function resolveCacheDir(workspacePath: string, configuredPath: string): string;
 export declare function packageRootFromImportMeta(importMetaUrl: string): string;
@@ -18,3 +25,4 @@ export declare function getDenseTransformersRuntime(cacheDir: string): Promise<{
     };
     pipeline: typeof import("@huggingface/transformers").pipeline;
 }>;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryformation/querylight-cli",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "Querylight CLI for building and querying local knowledge bases.",
   "license": "MIT",
   "homepage": "https://github.com/formation-res/querylight-cli#readme",

package/scripts/sparse-encode.py

@@ -88,7 +88,11 @@ def encode_documents(model_id: str, top_tokens: int, documents):
 
 
 def main():
-    payload = json.load(sys.stdin)
+    if len(sys.argv) > 1:
+        with open(sys.argv[1], encoding="utf-8") as handle:
+            payload = json.load(handle)
+    else:
+        payload = json.load(sys.stdin)
     action = payload["action"]
     model_id = payload["model_id"]
     if action == "download_only":