@tryformation/querylight-cli 0.2.0 → 0.2.1

package/README.md

@@ -208,8 +208,8 @@ The default workspace is `.kb/`.
   raw/
   normalized/
   indexes/
-    latest.json
-    latest.meta.json
+    latest.json.gz
+    latest.meta.json.gz
   runs/
   logs/
 ```

package/dist/cli/main.js

@@ -16,7 +16,7 @@ import path from "path";
 import YAML from "yaml";
 
 // src/core/constants.ts
-var PACKAGE_VERSION = "0.2.0";
+var PACKAGE_VERSION = "0.2.1";
 var DEFAULT_WORKSPACE = ".kb";
 var DEFAULT_SHARED_MODEL_CACHE_DIR = "~/.qli/models/huggingface";
 var LEGACY_WORKSPACE_MODEL_CACHE_DIR = ".kb/models/huggingface";
@@ -490,8 +490,40 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir3, readFile as readFile4, writeFile as writeFile3 } from "fs/promises";
+import { mkdir as mkdir3, rm, writeFile as writeFile4 } from "fs/promises";
 import path7 from "path";
+
+// src/core/gzip-json.ts
+import { readFile as readFile4, writeFile as writeFile3 } from "fs/promises";
+import { promisify } from "util";
+import { gunzip, gzip } from "zlib";
+var gzipAsync = promisify(gzip);
+var gunzipAsync = promisify(gunzip);
+async function writeGzipJson(filePath, value) {
+  const payload = JSON.stringify(value, null, 2);
+  await writeFile3(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+}
+async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
+  if (await fileExists(gzipPath)) {
+    const payload = await readFile4(gzipPath);
+    return JSON.parse((await gunzipAsync(payload)).toString("utf8"));
+  }
+  if (legacyPath && await fileExists(legacyPath)) {
+    return JSON.parse(await readFile4(legacyPath, "utf8"));
+  }
+  return JSON.parse(await readFile4(gzipPath, "utf8"));
+}
+async function resolveExistingGzipOrFilePath(gzipPath, legacyPath) {
+  if (await fileExists(gzipPath)) {
+    return gzipPath;
+  }
+  if (legacyPath && await fileExists(legacyPath)) {
+    return legacyPath;
+  }
+  return gzipPath;
+}
+
+// src/vector/store.ts
 function vectorsDir(workspacePath) {
   return path7.join(workspacePath, "vectors");
 }
@@ -499,15 +531,27 @@ function sharedModelStateDir() {
   return path7.join(resolveQliHomeDir(), "models", "status");
 }
 function denseVectorPath(workspacePath) {
-  return path7.join(vectorsDir(workspacePath), "dense.latest.json");
+  return path7.join(vectorsDir(workspacePath), "dense.latest.json.gz");
 }
 function denseMetaPath(workspacePath) {
-  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json.gz");
 }
 function sparseVectorPath(workspacePath) {
-  return path7.join(vectorsDir(workspacePath), "sparse.latest.json");
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.json.gz");
 }
 function sparseMetaPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.meta.json.gz");
+}
+function legacyDenseVectorPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "dense.latest.json");
+}
+function legacyDenseMetaPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+}
+function legacySparseVectorPath(workspacePath) {
+  return path7.join(vectorsDir(workspacePath), "sparse.latest.json");
+}
+function legacySparseMetaPath(workspacePath) {
   return path7.join(vectorsDir(workspacePath), "sparse.latest.meta.json");
 }
 function pullMarkerPath(type, workspacePath, modelId, cacheDir) {
@@ -523,29 +567,37 @@ function sparsePullMarker(workspacePath, modelId, cacheDir) {
 }
 async function writeDensePayload(workspacePath, payload) {
   await mkdir3(vectorsDir(workspacePath), { recursive: true });
-  await writeFile3(denseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile3(denseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(denseVectorPath(workspacePath), payload);
+  await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm(legacyDenseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readDensePayload(workspacePath) {
-  return JSON.parse(await readFile4(denseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(denseVectorPath(workspacePath), legacyDenseVectorPath(workspacePath));
 }
 async function writeSparsePayload(workspacePath, payload) {
   await mkdir3(vectorsDir(workspacePath), { recursive: true });
-  await writeFile3(sparseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile3(sparseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(sparseVectorPath(workspacePath), payload);
+  await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm(legacySparseVectorPath(workspacePath), { force: true }),
+    rm(legacySparseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readSparsePayload(workspacePath) {
-  return JSON.parse(await readFile4(sparseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(sparseVectorPath(workspacePath), legacySparseVectorPath(workspacePath));
 }
 async function writeDensePullMarker(workspacePath, model, value) {
   const markerPath = densePullMarker(workspacePath, model.modelId, model.cacheDir);
   await mkdir3(path7.dirname(markerPath), { recursive: true });
-  await writeFile3(markerPath, JSON.stringify(value, null, 2), "utf8");
+  await writeFile4(markerPath, JSON.stringify(value, null, 2), "utf8");
 }
 async function writeSparsePullMarker(workspacePath, model, value) {
   const markerPath = sparsePullMarker(workspacePath, model.modelId, model.cacheDir);
   await mkdir3(path7.dirname(markerPath), { recursive: true });
-  await writeFile3(markerPath, JSON.stringify(value, null, 2), "utf8");
+  await writeFile4(markerPath, JSON.stringify(value, null, 2), "utf8");
 }
 async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
   const denseCacheDir = resolveCacheDir(workspacePath, dense.cacheDir);
@@ -556,7 +608,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       modelId: dense.modelId,
       cacheDir: denseCacheDir,
       available: await fileExists(densePullMarker(workspacePath, dense.modelId, dense.cacheDir)),
-      artifactExists: await fileExists(denseVectorPath(workspacePath))
+      artifactExists: await fileExists(denseVectorPath(workspacePath)) || await fileExists(legacyDenseVectorPath(workspacePath))
     },
     sparse: {
       configured: sparse.enabled,
@@ -564,7 +616,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       cacheDir: sparseCacheDir,
       uvAvailable,
       available: await fileExists(sparsePullMarker(workspacePath, sparse.modelId, sparse.cacheDir)),
-      artifactExists: await fileExists(sparseVectorPath(workspacePath))
+      artifactExists: await fileExists(sparseVectorPath(workspacePath)) || await fileExists(legacySparseVectorPath(workspacePath))
     }
   };
 }
@@ -963,31 +1015,63 @@ async function getModelStatus(workspacePath, config) {
 }
 
 // src/index/index-store.ts
-import { readFile as readFile5, writeFile as writeFile4 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm2 } from "fs/promises";
 import path10 from "path";
+function versionedIndexPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.json.gz`);
+}
+function versionedLegacyIndexPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.json`);
+}
+function versionedMetaPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.meta.json.gz`);
+}
+function versionedLegacyMetaPath(workspacePath, stamp) {
+  return path10.join(workspacePath, "indexes", `${stamp}.meta.json`);
+}
+function latestIndexPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.json.gz");
+}
+function legacyLatestIndexPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.json");
+}
+function latestMetaPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.meta.json.gz");
+}
+function legacyLatestMetaPath(workspacePath) {
+  return path10.join(workspacePath, "indexes", "latest.meta.json");
+}
 async function writeIndexArtifacts({
   workspacePath,
   indexState,
   metadata
 }) {
   const stamp = metadata.createdAt.replace(/[:.]/g, "-");
-  const indexPath = path10.join(workspacePath, "indexes", `${stamp}.json`);
-  const metaPath = path10.join(workspacePath, "indexes", `${stamp}.meta.json`);
-  const latestIndexPath = path10.join(workspacePath, "indexes", "latest.json");
-  const latestMetaPath = path10.join(workspacePath, "indexes", "latest.meta.json");
-  const indexPayload = JSON.stringify(indexState, null, 2);
-  const metaPayload = JSON.stringify(metadata, null, 2);
-  await writeFile4(indexPath, indexPayload, "utf8");
-  await writeFile4(metaPath, metaPayload, "utf8");
-  await writeFile4(latestIndexPath, indexPayload, "utf8");
-  await writeFile4(latestMetaPath, metaPayload, "utf8");
-  return { indexPath: latestIndexPath, metadataPath: latestMetaPath };
+  const indexPath = versionedIndexPath(workspacePath, stamp);
+  const metaPath = versionedMetaPath(workspacePath, stamp);
+  const latestIndexArtifactPath = latestIndexPath(workspacePath);
+  const latestMetadataArtifactPath = latestMetaPath(workspacePath);
+  await mkdir6(path10.join(workspacePath, "indexes"), { recursive: true });
+  await writeGzipJson(indexPath, indexState);
+  await writeGzipJson(metaPath, metadata);
+  await writeGzipJson(latestIndexArtifactPath, indexState);
+  await writeGzipJson(latestMetadataArtifactPath, metadata);
+  await Promise.all([
+    rm2(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm2(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm2(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm2(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+  ]);
+  return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }
 async function readLatestIndexState(workspacePath) {
-  return JSON.parse(await readFile5(path10.join(workspacePath, "indexes", "latest.json"), "utf8"));
+  return readJsonFromGzipOrFile(latestIndexPath(workspacePath), legacyLatestIndexPath(workspacePath));
 }
 async function readLatestIndexMetadata(workspacePath) {
-  return JSON.parse(await readFile5(path10.join(workspacePath, "indexes", "latest.meta.json"), "utf8"));
+  return readJsonFromGzipOrFile(latestMetaPath(workspacePath), legacyLatestMetaPath(workspacePath));
+}
+async function resolveLatestIndexArtifactPath(workspacePath) {
+  return resolveExistingGzipOrFilePath(latestIndexPath(workspacePath), legacyLatestIndexPath(workspacePath));
 }
 
 // src/index/querylight-indexer.ts
@@ -1178,7 +1262,7 @@ async function removeSource(workspacePath, sourceId) {
 }
 
 // src/ingest/document-utils.ts
-import { mkdir as mkdir6, rm, writeFile as writeFile5 } from "fs/promises";
+import { mkdir as mkdir7, rm as rm3, writeFile as writeFile5 } from "fs/promises";
 import path14 from "path";
 
 // src/normalize/normalize-markdown.ts
@@ -1231,7 +1315,7 @@ async function writeNormalizedDocument({
   normalizedPath,
   markdown
 }) {
-  await mkdir6(path14.dirname(normalizedPath), { recursive: true });
+  await mkdir7(path14.dirname(normalizedPath), { recursive: true });
   await writeFile5(
     normalizedPath,
     withFrontmatter(
@@ -1254,8 +1338,8 @@ async function writeNormalizedDocument({
 }
 async function deleteDocumentArtifacts(document) {
   await Promise.all([
-    document.rawPath ? rm(document.rawPath, { force: true }) : Promise.resolve(),
-    rm(document.normalizedPath, { force: true })
+    document.rawPath ? rm3(document.rawPath, { force: true }) : Promise.resolve(),
+    rm3(document.normalizedPath, { force: true })
   ]);
 }
 
@@ -1279,7 +1363,7 @@ async function listDirectoryFiles(source) {
 
 // src/ingest/adapters/file-adapter.ts
 import { basename, extname, resolve } from "path";
-import { mkdir as mkdir7, readFile as readFile9, stat as stat3, writeFile as writeFile6 } from "fs/promises";
+import { mkdir as mkdir8, readFile as readFile8, stat as stat3, writeFile as writeFile6 } from "fs/promises";
 
 // src/ingest/extractors/docx-extractor.ts
 import mammoth from "mammoth";
@@ -1453,16 +1537,16 @@ function extractPublicationDateFromHtml(html) {
 }
 
 // src/ingest/extractors/markdown-extractor.ts
-import { readFile as readFile6 } from "fs/promises";
+import { readFile as readFile5 } from "fs/promises";
 async function extractMarkdown(filePath) {
-  return readFile6(filePath, "utf8");
+  return readFile5(filePath, "utf8");
 }
 
 // src/ingest/extractors/pdf-extractor.ts
-import { readFile as readFile7 } from "fs/promises";
+import { readFile as readFile6 } from "fs/promises";
 import { PDFParse } from "pdf-parse";
 async function extractPdf(filePath) {
-  const buffer = await readFile7(filePath);
+  const buffer = await readFile6(filePath);
   const parser = new PDFParse({ data: buffer });
   try {
     const parsed = await parser.getText();
@@ -1473,9 +1557,9 @@ async function extractPdf(filePath) {
 }
 
 // src/ingest/extractors/text-extractor.ts
-import { readFile as readFile8 } from "fs/promises";
+import { readFile as readFile7 } from "fs/promises";
 async function extractText(filePath) {
-  return readFile8(filePath, "utf8");
+  return readFile7(filePath, "utf8");
 }
 
 // src/ingest/adapters/file-adapter.ts
@@ -1510,7 +1594,7 @@ async function extractFileContent(filePath, mimeType) {
 ${text}`, raw: text };
   }
   if (mimeType === "text/html") {
-    const raw = await readFile9(filePath, "utf8");
+    const raw = await readFile8(filePath, "utf8");
     const extracted = extractHtmlToMarkdown(raw);
     return { title: extracted.title, markdown: `# ${extracted.title}
 
@@ -1566,8 +1650,8 @@ async function ingestFile({
   const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
   const indexedAt = now;
   const crawledAt = now;
-  await mkdir7(resolve(workspacePath, "normalized"), { recursive: true });
-  await mkdir7(resolve(workspacePath, "raw", source.id), { recursive: true });
+  await mkdir8(resolve(workspacePath, "normalized"), { recursive: true });
+  await mkdir8(resolve(workspacePath, "raw", source.id), { recursive: true });
   if (extracted.raw) {
     await writeFile6(rawPath, extracted.raw, "utf8");
   }
@@ -1632,7 +1716,7 @@ ${content}`;
   const now = (/* @__PURE__ */ new Date()).toISOString();
   const lastChangedAt = previous?.contentHash === contentHash ? previous.lastChangedAt : now;
   const indexedAt = now;
-  await mkdir7(resolve(workspacePath, "normalized"), { recursive: true });
+  await mkdir8(resolve(workspacePath, "normalized"), { recursive: true });
   await writeNormalizedDocument({
     documentId,
     sourceId: source.id,
@@ -1676,7 +1760,7 @@ async function reprocessStoredDocument(document, source) {
   if (!document.rawPath) {
     return null;
   }
-  const raw = await readFile9(document.rawPath, "utf8");
+  const raw = await readFile8(document.rawPath, "utf8");
   const fallbackTitle = document.title || basename(document.uri);
   const extracted = await extractRawContent(raw, document.mimeType, fallbackTitle);
   const contentHash = sha256(extracted.markdown);
@@ -1793,7 +1877,7 @@ async function parseRssFeedDocument(xml, source) {
 }
 
 // src/ingest/adapters/url-adapter.ts
-import { mkdir as mkdir8, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir9, readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
 import path16 from "path";
 
 // src/core/urls.ts
@@ -1845,7 +1929,7 @@ ${extracted.markdown}`;
   const indexedAt = now;
   const crawledAt = now;
   const resolvedPublicationDate = choosePublicationDate(publicationDate, extractPublicationDateFromHtml(body), previous?.publicationDate);
-  await mkdir8(path16.resolve(workspacePath, "raw", source.id), { recursive: true });
+  await mkdir9(path16.resolve(workspacePath, "raw", source.id), { recursive: true });
   await writeFile7(rawPath, body, "utf8");
   await writeNormalizedDocument({
     documentId,
@@ -1966,7 +2050,7 @@ async function reprocessRemoteDocument(document, source) {
   if (!document.rawPath || !await fileExists(document.rawPath)) {
     return null;
   }
-  const raw = await readFile10(document.rawPath, "utf8");
+  const raw = await readFile9(document.rawPath, "utf8");
   const extracted = extractHtmlToMarkdown(raw);
   const markdown = `# ${extracted.title}
 
@@ -2684,7 +2768,7 @@ async function discoverWebsiteFeed(websiteUrl, userAgent) {
 }
 
 // src/query/search-service.ts
-import { readFile as readFile11 } from "fs/promises";
+import { readFile as readFile10 } from "fs/promises";
 import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
 import path18 from "path";
 async function loadHydratedIndex(workspacePath) {
@@ -2914,7 +2998,7 @@ async function buildSnippetWithAdjacentChunks(chunk, query, {
     if (!await fileExists(document.normalizedPath)) {
       return buildSnippet(chunk.text, query);
     }
-    const raw = await readFile11(document.normalizedPath, "utf8");
+    const raw = await readFile10(document.normalizedPath, "utf8");
     orderedChunks = buildChunksForDocument(document, raw, config);
     orderedChunkCache.set(document.id, orderedChunks);
   }
@@ -4210,7 +4294,7 @@ Examples:
     try {
       const meta = await readLatestIndexMetadata(workspace);
       latestIndex = meta.createdAt;
-      indexSize = (await stat4(`${workspace}/indexes/latest.json`)).size;
+      indexSize = (await stat4(await resolveLatestIndexArtifactPath(workspace))).size;
     } catch {
       latestIndex = void 0;
     }

package/dist/core/constants.d.ts

@@ -1,5 +1,5 @@
 export declare const PACKAGE_NAME = "@tryformation/querylight-cli";
-export declare const PACKAGE_VERSION = "0.2.0";
+export declare const PACKAGE_VERSION = "0.2.1";
 export declare const DEFAULT_WORKSPACE = ".kb";
 export declare const DEFAULT_SHARED_MODEL_CACHE_DIR = "~/.qli/models/huggingface";
 export declare const LEGACY_WORKSPACE_MODEL_CACHE_DIR = ".kb/models/huggingface";

package/dist/core/gzip-json.d.ts

@@ -0,0 +1,3 @@
+export declare function writeGzipJson(filePath: string, value: unknown): Promise<void>;
+export declare function readJsonFromGzipOrFile<T>(gzipPath: string, legacyPath?: string): Promise<T>;
+export declare function resolveExistingGzipOrFilePath(gzipPath: string, legacyPath?: string): Promise<string>;

package/dist/index/index-store.d.ts

@@ -1,4 +1,6 @@
 import type { IndexMetadata } from "../types/models.js";
+export declare function latestIndexPath(workspacePath: string): string;
+export declare function latestMetaPath(workspacePath: string): string;
 export declare function writeIndexArtifacts({ workspacePath, indexState, metadata }: {
     workspacePath: string;
     indexState: object;
@@ -9,3 +11,4 @@ export declare function writeIndexArtifacts({ workspacePath, indexState, metadat
 }>;
 export declare function readLatestIndexState(workspacePath: string): Promise<object>;
 export declare function readLatestIndexMetadata(workspacePath: string): Promise<IndexMetadata>;
+export declare function resolveLatestIndexArtifactPath(workspacePath: string): Promise<string>;

package/dist/index.js

@@ -1885,8 +1885,31 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir6, readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm2, writeFile as writeFile7 } from "fs/promises";
 import path13 from "path";
+
+// src/core/gzip-json.ts
+import { readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
+import { promisify } from "util";
+import { gunzip, gzip } from "zlib";
+var gzipAsync = promisify(gzip);
+var gunzipAsync = promisify(gunzip);
+async function writeGzipJson(filePath, value) {
+  const payload = JSON.stringify(value, null, 2);
+  await writeFile6(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+}
+async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
+  if (await fileExists(gzipPath)) {
+    const payload = await readFile9(gzipPath);
+    return JSON.parse((await gunzipAsync(payload)).toString("utf8"));
+  }
+  if (legacyPath && await fileExists(legacyPath)) {
+    return JSON.parse(await readFile9(legacyPath, "utf8"));
+  }
+  return JSON.parse(await readFile9(gzipPath, "utf8"));
+}
+
+// src/vector/store.ts
 function vectorsDir(workspacePath) {
   return path13.join(workspacePath, "vectors");
 }
@@ -1894,15 +1917,27 @@ function sharedModelStateDir() {
   return path13.join(resolveQliHomeDir(), "models", "status");
 }
 function denseVectorPath(workspacePath) {
-  return path13.join(vectorsDir(workspacePath), "dense.latest.json");
+  return path13.join(vectorsDir(workspacePath), "dense.latest.json.gz");
 }
 function denseMetaPath(workspacePath) {
-  return path13.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+  return path13.join(vectorsDir(workspacePath), "dense.latest.meta.json.gz");
 }
 function sparseVectorPath(workspacePath) {
-  return path13.join(vectorsDir(workspacePath), "sparse.latest.json");
+  return path13.join(vectorsDir(workspacePath), "sparse.latest.json.gz");
 }
 function sparseMetaPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "sparse.latest.meta.json.gz");
+}
+function legacyDenseVectorPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "dense.latest.json");
+}
+function legacyDenseMetaPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "dense.latest.meta.json");
+}
+function legacySparseVectorPath(workspacePath) {
+  return path13.join(vectorsDir(workspacePath), "sparse.latest.json");
+}
+function legacySparseMetaPath(workspacePath) {
   return path13.join(vectorsDir(workspacePath), "sparse.latest.meta.json");
 }
 function pullMarkerPath(type, workspacePath, modelId, cacheDir) {
@@ -1918,19 +1953,27 @@ function sparsePullMarker(workspacePath, modelId, cacheDir) {
 }
 async function writeDensePayload(workspacePath, payload) {
   await mkdir6(vectorsDir(workspacePath), { recursive: true });
-  await writeFile6(denseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile6(denseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(denseVectorPath(workspacePath), payload);
+  await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm2(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm2(legacyDenseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readDensePayload(workspacePath) {
-  return JSON.parse(await readFile9(denseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(denseVectorPath(workspacePath), legacyDenseVectorPath(workspacePath));
 }
 async function writeSparsePayload(workspacePath, payload) {
   await mkdir6(vectorsDir(workspacePath), { recursive: true });
-  await writeFile6(sparseVectorPath(workspacePath), JSON.stringify(payload, null, 2), "utf8");
-  await writeFile6(sparseMetaPath(workspacePath), JSON.stringify(payload.metadata, null, 2), "utf8");
+  await writeGzipJson(sparseVectorPath(workspacePath), payload);
+  await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
+  await Promise.all([
+    rm2(legacySparseVectorPath(workspacePath), { force: true }),
+    rm2(legacySparseMetaPath(workspacePath), { force: true })
+  ]);
 }
 async function readSparsePayload(workspacePath) {
-  return JSON.parse(await readFile9(sparseVectorPath(workspacePath), "utf8"));
+  return readJsonFromGzipOrFile(sparseVectorPath(workspacePath), legacySparseVectorPath(workspacePath));
 }
 async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
   const denseCacheDir = resolveCacheDir(workspacePath, dense.cacheDir);
@@ -1941,7 +1984,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       modelId: dense.modelId,
       cacheDir: denseCacheDir,
       available: await fileExists(densePullMarker(workspacePath, dense.modelId, dense.cacheDir)),
-      artifactExists: await fileExists(denseVectorPath(workspacePath))
+      artifactExists: await fileExists(denseVectorPath(workspacePath)) || await fileExists(legacyDenseVectorPath(workspacePath))
     },
     sparse: {
       configured: sparse.enabled,
@@ -1949,7 +1992,7 @@ async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
       cacheDir: sparseCacheDir,
       uvAvailable,
       available: await fileExists(sparsePullMarker(workspacePath, sparse.modelId, sparse.cacheDir)),
-      artifactExists: await fileExists(sparseVectorPath(workspacePath))
+      artifactExists: await fileExists(sparseVectorPath(workspacePath)) || await fileExists(legacySparseVectorPath(workspacePath))
     }
   };
 }
@@ -2266,28 +2309,57 @@ async function buildVectorArtifacts({
 }
 
 // src/index/index-store.ts
-import { readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir9, rm as rm3 } from "fs/promises";
 import path16 from "path";
+function versionedIndexPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.json.gz`);
+}
+function versionedLegacyIndexPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.json`);
+}
+function versionedMetaPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.meta.json.gz`);
+}
+function versionedLegacyMetaPath(workspacePath, stamp) {
+  return path16.join(workspacePath, "indexes", `${stamp}.meta.json`);
+}
+function latestIndexPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.json.gz");
+}
+function legacyLatestIndexPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.json");
+}
+function latestMetaPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.meta.json.gz");
+}
+function legacyLatestMetaPath(workspacePath) {
+  return path16.join(workspacePath, "indexes", "latest.meta.json");
+}
 async function writeIndexArtifacts({
   workspacePath,
   indexState,
   metadata
 }) {
   const stamp = metadata.createdAt.replace(/[:.]/g, "-");
-  const indexPath = path16.join(workspacePath, "indexes", `${stamp}.json`);
-  const metaPath = path16.join(workspacePath, "indexes", `${stamp}.meta.json`);
-  const latestIndexPath = path16.join(workspacePath, "indexes", "latest.json");
-  const latestMetaPath = path16.join(workspacePath, "indexes", "latest.meta.json");
-  const indexPayload = JSON.stringify(indexState, null, 2);
-  const metaPayload = JSON.stringify(metadata, null, 2);
-  await writeFile7(indexPath, indexPayload, "utf8");
-  await writeFile7(metaPath, metaPayload, "utf8");
-  await writeFile7(latestIndexPath, indexPayload, "utf8");
-  await writeFile7(latestMetaPath, metaPayload, "utf8");
-  return { indexPath: latestIndexPath, metadataPath: latestMetaPath };
+  const indexPath = versionedIndexPath(workspacePath, stamp);
+  const metaPath = versionedMetaPath(workspacePath, stamp);
+  const latestIndexArtifactPath = latestIndexPath(workspacePath);
+  const latestMetadataArtifactPath = latestMetaPath(workspacePath);
+  await mkdir9(path16.join(workspacePath, "indexes"), { recursive: true });
+  await writeGzipJson(indexPath, indexState);
+  await writeGzipJson(metaPath, metadata);
+  await writeGzipJson(latestIndexArtifactPath, indexState);
+  await writeGzipJson(latestMetadataArtifactPath, metadata);
+  await Promise.all([
+    rm3(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm3(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm3(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm3(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+  ]);
+  return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }
 async function readLatestIndexState(workspacePath) {
-  return JSON.parse(await readFile10(path16.join(workspacePath, "indexes", "latest.json"), "utf8"));
+  return readJsonFromGzipOrFile(latestIndexPath(workspacePath), legacyLatestIndexPath(workspacePath));
 }
 
 // src/index/querylight-indexer.ts
@@ -2387,7 +2459,7 @@ async function buildIndex({
 }
 
 // src/query/search-service.ts
-import { readFile as readFile11 } from "fs/promises";
+import { readFile as readFile10 } from "fs/promises";
 import { BoolQuery, MatchQuery, OP, TermQuery, reciprocalRankFusion } from "@tryformation/querylight-ts";
 import path18 from "path";
 async function loadHydratedIndex(workspacePath) {
@@ -2617,7 +2689,7 @@ async function buildSnippetWithAdjacentChunks(chunk, query, {
     if (!await fileExists(document.normalizedPath)) {
       return buildSnippet(chunk.text, query);
     }
-    const raw = await readFile11(document.normalizedPath, "utf8");
+    const raw = await readFile10(document.normalizedPath, "utf8");
     orderedChunks = buildChunksForDocument(document, raw, config);
     orderedChunkCache.set(document.id, orderedChunks);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryformation/querylight-cli",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Querylight CLI for building and querying local knowledge bases.",
   "license": "MIT",
   "homepage": "https://github.com/formation-res/querylight-cli#readme",