@tryformation/querylight-cli 0.2.1 → 0.2.2

package/dist/cli/main.js

@@ -387,6 +387,7 @@ import os from "os";
 import path6 from "path";
 import { fileURLToPath } from "url";
 import { execFile, execFileSync } from "child_process";
+import { mkdtemp, rm, writeFile as writeFile3 } from "fs/promises";
 
 // src/core/files.ts
 import { stat as stat2 } from "fs/promises";
@@ -400,6 +401,7 @@ async function fileExists(filePath) {
 }
 
 // src/vector/runtime.ts
+var sparseExecFileSync = execFileSync;
 function resolveQliHomeDir() {
   return path6.resolve(process.env.QLI_HOME ?? path6.join(os.homedir(), ".qli"));
 }
@@ -455,29 +457,36 @@ async function runSparsePython({
 }) {
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
   const scriptPath = await sparseScriptPath(importMetaUrl);
-  return execFileSync(
-    "uv",
-    [
-      "run",
-      "--with",
-      "torch",
-      "--with",
-      "transformers",
-      "--with",
-      "huggingface_hub",
-      "python",
-      scriptPath
-    ],
-    {
-      encoding: "utf8",
-      maxBuffer: 1024 * 1024 * 1024,
-      input: JSON.stringify(payload),
-      env: {
-        ...process.env,
-        HF_HOME: cacheDir
+  const payloadDir = await mkdtemp(path6.join(os.tmpdir(), "qli-sparse-"));
+  const payloadPath = path6.join(payloadDir, "payload.json");
+  await writeFile3(payloadPath, JSON.stringify(payload), "utf8");
+  try {
+    return sparseExecFileSync(
+      "uv",
+      [
+        "run",
+        "--with",
+        "torch",
+        "--with",
+        "transformers",
+        "--with",
+        "huggingface_hub",
+        "python",
+        scriptPath,
+        payloadPath
+      ],
+      {
+        encoding: "utf8",
+        maxBuffer: 1024 * 1024 * 1024,
+        env: {
+          ...process.env,
+          HF_HOME: cacheDir
+        }
       }
-    }
-  );
+    );
+  } finally {
+    await rm(payloadDir, { recursive: true, force: true });
+  }
 }
 async function getDenseTransformersRuntime(cacheDir) {
   const transformers = await import("@huggingface/transformers");
@@ -490,18 +499,18 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir3, rm, writeFile as writeFile4 } from "fs/promises";
+import { mkdir as mkdir3, rm as rm2, writeFile as writeFile5 } from "fs/promises";
 import path7 from "path";
 
 // src/core/gzip-json.ts
-import { readFile as readFile4, writeFile as writeFile3 } from "fs/promises";
+import { readFile as readFile4, writeFile as writeFile4 } from "fs/promises";
 import { promisify } from "util";
 import { gunzip, gzip } from "zlib";
 var gzipAsync = promisify(gzip);
 var gunzipAsync = promisify(gunzip);
 async function writeGzipJson(filePath, value) {
   const payload = JSON.stringify(value, null, 2);
-  await writeFile3(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+  await writeFile4(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
 }
 async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
   if (await fileExists(gzipPath)) {
@@ -570,8 +579,8 @@ async function writeDensePayload(workspacePath, payload) {
   await writeGzipJson(denseVectorPath(workspacePath), payload);
   await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
   await Promise.all([
-    rm(legacyDenseVectorPath(workspacePath), { force: true }),
-    rm(legacyDenseMetaPath(workspacePath), { force: true })
+    rm2(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm2(legacyDenseMetaPath(workspacePath), { force: true })
   ]);
 }
 async function readDensePayload(workspacePath) {
@@ -582,8 +591,8 @@ async function writeSparsePayload(workspacePath, payload) {
   await writeGzipJson(sparseVectorPath(workspacePath), payload);
   await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
   await Promise.all([
-    rm(legacySparseVectorPath(workspacePath), { force: true }),
-    rm(legacySparseMetaPath(workspacePath), { force: true })
+    rm2(legacySparseVectorPath(workspacePath), { force: true }),
+    rm2(legacySparseMetaPath(workspacePath), { force: true })
   ]);
 }
 async function readSparsePayload(workspacePath) {
@@ -592,12 +601,12 @@ async function readSparsePayload(workspacePath) {
 async function writeDensePullMarker(workspacePath, model, value) {
   const markerPath = densePullMarker(workspacePath, model.modelId, model.cacheDir);
   await mkdir3(path7.dirname(markerPath), { recursive: true });
-  await writeFile4(markerPath, JSON.stringify(value, null, 2), "utf8");
+  await writeFile5(markerPath, JSON.stringify(value, null, 2), "utf8");
 }
 async function writeSparsePullMarker(workspacePath, model, value) {
   const markerPath = sparsePullMarker(workspacePath, model.modelId, model.cacheDir);
   await mkdir3(path7.dirname(markerPath), { recursive: true });
-  await writeFile4(markerPath, JSON.stringify(value, null, 2), "utf8");
+  await writeFile5(markerPath, JSON.stringify(value, null, 2), "utf8");
 }
 async function buildModelStatus(workspacePath, dense, sparse, uvAvailable) {
   const denseCacheDir = resolveCacheDir(workspacePath, dense.cacheDir);
@@ -1015,7 +1024,7 @@ async function getModelStatus(workspacePath, config) {
 }
 
 // src/index/index-store.ts
-import { mkdir as mkdir6, rm as rm2 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm3 } from "fs/promises";
 import path10 from "path";
 function versionedIndexPath(workspacePath, stamp) {
   return path10.join(workspacePath, "indexes", `${stamp}.json.gz`);
@@ -1057,10 +1066,10 @@ async function writeIndexArtifacts({
   await writeGzipJson(latestIndexArtifactPath, indexState);
   await writeGzipJson(latestMetadataArtifactPath, metadata);
   await Promise.all([
-    rm2(legacyLatestIndexPath(workspacePath), { force: true }),
-    rm2(legacyLatestMetaPath(workspacePath), { force: true }),
-    rm2(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
-    rm2(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+    rm3(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm3(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm3(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm3(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
   ]);
   return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }
@@ -1262,7 +1271,7 @@ async function removeSource(workspacePath, sourceId) {
 }
 
 // src/ingest/document-utils.ts
-import { mkdir as mkdir7, rm as rm3, writeFile as writeFile5 } from "fs/promises";
+import { mkdir as mkdir7, rm as rm4, writeFile as writeFile6 } from "fs/promises";
 import path14 from "path";
 
 // src/normalize/normalize-markdown.ts
@@ -1316,7 +1325,7 @@ async function writeNormalizedDocument({
   markdown
 }) {
   await mkdir7(path14.dirname(normalizedPath), { recursive: true });
-  await writeFile5(
+  await writeFile6(
     normalizedPath,
     withFrontmatter(
       {
@@ -1338,8 +1347,8 @@ async function writeNormalizedDocument({
 }
 async function deleteDocumentArtifacts(document) {
   await Promise.all([
-    document.rawPath ? rm3(document.rawPath, { force: true }) : Promise.resolve(),
-    rm3(document.normalizedPath, { force: true })
+    document.rawPath ? rm4(document.rawPath, { force: true }) : Promise.resolve(),
+    rm4(document.normalizedPath, { force: true })
   ]);
 }
 
@@ -1363,7 +1372,7 @@ async function listDirectoryFiles(source) {
 
 // src/ingest/adapters/file-adapter.ts
 import { basename, extname, resolve } from "path";
-import { mkdir as mkdir8, readFile as readFile8, stat as stat3, writeFile as writeFile6 } from "fs/promises";
+import { mkdir as mkdir8, readFile as readFile8, stat as stat3, writeFile as writeFile7 } from "fs/promises";
 
 // src/ingest/extractors/docx-extractor.ts
 import mammoth from "mammoth";
@@ -1653,7 +1662,7 @@ async function ingestFile({
   await mkdir8(resolve(workspacePath, "normalized"), { recursive: true });
   await mkdir8(resolve(workspacePath, "raw", source.id), { recursive: true });
   if (extracted.raw) {
-    await writeFile6(rawPath, extracted.raw, "utf8");
+    await writeFile7(rawPath, extracted.raw, "utf8");
   }
   await writeNormalizedDocument({
     documentId,
@@ -1877,7 +1886,7 @@ async function parseRssFeedDocument(xml, source) {
 }
 
 // src/ingest/adapters/url-adapter.ts
-import { mkdir as mkdir9, readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir9, readFile as readFile9, writeFile as writeFile8 } from "fs/promises";
 import path16 from "path";
 
 // src/core/urls.ts
@@ -1930,7 +1939,7 @@ ${extracted.markdown}`;
   const crawledAt = now;
   const resolvedPublicationDate = choosePublicationDate(publicationDate, extractPublicationDateFromHtml(body), previous?.publicationDate);
   await mkdir9(path16.resolve(workspacePath, "raw", source.id), { recursive: true });
-  await writeFile7(rawPath, body, "utf8");
+  await writeFile8(rawPath, body, "utf8");
   await writeNormalizedDocument({
     documentId,
     sourceId: source.id,

package/dist/index.js

@@ -1795,6 +1795,8 @@ import os from "os";
 import path12 from "path";
 import { fileURLToPath } from "url";
 import { execFile, execFileSync } from "child_process";
+import { mkdtemp, rm as rm2, writeFile as writeFile6 } from "fs/promises";
+var sparseExecFileSync = execFileSync;
 function resolveQliHomeDir() {
   return path12.resolve(process.env.QLI_HOME ?? path12.join(os.homedir(), ".qli"));
 }
@@ -1850,29 +1852,36 @@ async function runSparsePython({
 }) {
   const cacheDir = resolveCacheDir(workspacePath, config.cacheDir);
   const scriptPath = await sparseScriptPath(importMetaUrl);
-  return execFileSync(
-    "uv",
-    [
-      "run",
-      "--with",
-      "torch",
-      "--with",
-      "transformers",
-      "--with",
-      "huggingface_hub",
-      "python",
-      scriptPath
-    ],
-    {
-      encoding: "utf8",
-      maxBuffer: 1024 * 1024 * 1024,
-      input: JSON.stringify(payload),
-      env: {
-        ...process.env,
-        HF_HOME: cacheDir
+  const payloadDir = await mkdtemp(path12.join(os.tmpdir(), "qli-sparse-"));
+  const payloadPath = path12.join(payloadDir, "payload.json");
+  await writeFile6(payloadPath, JSON.stringify(payload), "utf8");
+  try {
+    return sparseExecFileSync(
+      "uv",
+      [
+        "run",
+        "--with",
+        "torch",
+        "--with",
+        "transformers",
+        "--with",
+        "huggingface_hub",
+        "python",
+        scriptPath,
+        payloadPath
+      ],
+      {
+        encoding: "utf8",
+        maxBuffer: 1024 * 1024 * 1024,
+        env: {
+          ...process.env,
+          HF_HOME: cacheDir
+        }
       }
-    }
-  );
+    );
+  } finally {
+    await rm2(payloadDir, { recursive: true, force: true });
+  }
 }
 async function getDenseTransformersRuntime(cacheDir) {
   const transformers = await import("@huggingface/transformers");
@@ -1885,18 +1894,18 @@ async function getDenseTransformersRuntime(cacheDir) {
 }
 
 // src/vector/store.ts
-import { mkdir as mkdir6, rm as rm2, writeFile as writeFile7 } from "fs/promises";
+import { mkdir as mkdir6, rm as rm3, writeFile as writeFile8 } from "fs/promises";
 import path13 from "path";
 
 // src/core/gzip-json.ts
-import { readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
+import { readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
 import { promisify } from "util";
 import { gunzip, gzip } from "zlib";
 var gzipAsync = promisify(gzip);
 var gunzipAsync = promisify(gunzip);
 async function writeGzipJson(filePath, value) {
   const payload = JSON.stringify(value, null, 2);
-  await writeFile6(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
+  await writeFile7(filePath, await gzipAsync(Buffer.from(payload, "utf8")));
 }
 async function readJsonFromGzipOrFile(gzipPath, legacyPath) {
   if (await fileExists(gzipPath)) {
@@ -1956,8 +1965,8 @@ async function writeDensePayload(workspacePath, payload) {
   await writeGzipJson(denseVectorPath(workspacePath), payload);
   await writeGzipJson(denseMetaPath(workspacePath), payload.metadata);
   await Promise.all([
-    rm2(legacyDenseVectorPath(workspacePath), { force: true }),
-    rm2(legacyDenseMetaPath(workspacePath), { force: true })
+    rm3(legacyDenseVectorPath(workspacePath), { force: true }),
+    rm3(legacyDenseMetaPath(workspacePath), { force: true })
   ]);
 }
 async function readDensePayload(workspacePath) {
@@ -1968,8 +1977,8 @@ async function writeSparsePayload(workspacePath, payload) {
   await writeGzipJson(sparseVectorPath(workspacePath), payload);
   await writeGzipJson(sparseMetaPath(workspacePath), payload.metadata);
   await Promise.all([
-    rm2(legacySparseVectorPath(workspacePath), { force: true }),
-    rm2(legacySparseMetaPath(workspacePath), { force: true })
+    rm3(legacySparseVectorPath(workspacePath), { force: true }),
+    rm3(legacySparseMetaPath(workspacePath), { force: true })
   ]);
 }
 async function readSparsePayload(workspacePath) {
@@ -2309,7 +2318,7 @@ async function buildVectorArtifacts({
 }
 
 // src/index/index-store.ts
-import { mkdir as mkdir9, rm as rm3 } from "fs/promises";
+import { mkdir as mkdir9, rm as rm4 } from "fs/promises";
 import path16 from "path";
 function versionedIndexPath(workspacePath, stamp) {
   return path16.join(workspacePath, "indexes", `${stamp}.json.gz`);
@@ -2351,10 +2360,10 @@ async function writeIndexArtifacts({
   await writeGzipJson(latestIndexArtifactPath, indexState);
   await writeGzipJson(latestMetadataArtifactPath, metadata);
   await Promise.all([
-    rm3(legacyLatestIndexPath(workspacePath), { force: true }),
-    rm3(legacyLatestMetaPath(workspacePath), { force: true }),
-    rm3(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
-    rm3(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
+    rm4(legacyLatestIndexPath(workspacePath), { force: true }),
+    rm4(legacyLatestMetaPath(workspacePath), { force: true }),
+    rm4(versionedLegacyIndexPath(workspacePath, stamp), { force: true }),
+    rm4(versionedLegacyMetaPath(workspacePath, stamp), { force: true })
   ]);
   return { indexPath: latestIndexArtifactPath, metadataPath: latestMetadataArtifactPath };
 }

package/dist/vector/runtime.d.ts

@@ -1,4 +1,11 @@
 import type { SparseVectorModelConfig } from "../types/models.js";
+type SparseExecOptions = {
+    encoding: BufferEncoding;
+    maxBuffer: number;
+    env: NodeJS.ProcessEnv;
+};
+type SparseExecFileSync = (file: string, args: string[], options: SparseExecOptions) => string;
+export declare function setSparseExecFileSyncForTests(fn: SparseExecFileSync | null): void;
 export declare function resolveQliHomeDir(): string;
 export declare function resolveCacheDir(workspacePath: string, configuredPath: string): string;
 export declare function packageRootFromImportMeta(importMetaUrl: string): string;
@@ -18,3 +25,4 @@ export declare function getDenseTransformersRuntime(cacheDir: string): Promise<{
     };
     pipeline: typeof import("@huggingface/transformers").pipeline;
 }>;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryformation/querylight-cli",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Querylight CLI for building and querying local knowledge bases.",
   "license": "MIT",
   "homepage": "https://github.com/formation-res/querylight-cli#readme",

package/scripts/sparse-encode.py

@@ -88,7 +88,11 @@ def encode_documents(model_id: str, top_tokens: int, documents):
 
 
 def main():
-    payload = json.load(sys.stdin)
+    if len(sys.argv) > 1:
+        with open(sys.argv[1], encoding="utf-8") as handle:
+            payload = json.load(handle)
+    else:
+        payload = json.load(sys.stdin)
     action = payload["action"]
     model_id = payload["model_id"]
     if action == "download_only":