@larkup/cli 0.2.31 → 0.3.0

package/dist/{dist-es-LWW2DNRF.js → dist-es-LH5Y4P7D.js}

@@ -14,11 +14,11 @@ import {
 } from "./chunk-MACWQEJS.js";
 import "./chunk-QGM4M3NI.js";
 
-// ../../node_modules/.pnpm/@aws-sdk+s3-request-presigner@3.1125.0/node_modules/@aws-sdk/s3-request-presigner/dist-es/constants.js
+// ../../node_modules/.pnpm/@aws-sdk+s3-request-presigner@3.1127.0/node_modules/@aws-sdk/s3-request-presigner/dist-es/constants.js
 var UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
 var SHA256_HEADER = "X-Amz-Content-Sha256";
 
-// ../../node_modules/.pnpm/@aws-sdk+s3-request-presigner@3.1125.0/node_modules/@aws-sdk/s3-request-presigner/dist-es/presigner.js
+// ../../node_modules/.pnpm/@aws-sdk+s3-request-presigner@3.1127.0/node_modules/@aws-sdk/s3-request-presigner/dist-es/presigner.js
 var S3RequestPresigner = class {
   signer;
   constructor(options) {
@@ -73,7 +73,7 @@ var S3RequestPresigner = class {
   }
 };
 
-// ../../node_modules/.pnpm/@aws-sdk+s3-request-presigner@3.1125.0/node_modules/@aws-sdk/s3-request-presigner/dist-es/getSignedUrl.js
+// ../../node_modules/.pnpm/@aws-sdk+s3-request-presigner@3.1127.0/node_modules/@aws-sdk/s3-request-presigner/dist-es/getSignedUrl.js
 var getSignedUrl = async (client, command, options = {}) => {
   let s3Presigner;
   let region;

package/dist/index.js

@@ -36,7 +36,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "@larkup/cli",
-  version: "0.2.31",
+  version: "0.3.0",
   engines: {
     node: ">=22"
   },
@@ -57,25 +57,28 @@ var package_default = {
     "type-check": "tsc --noEmit"
   },
   dependencies: {
-    "@ai-sdk/cohere": "^4.0.35",
-    "@ai-sdk/deepseek": "^3.0.37",
-    "@ai-sdk/gateway": "^4.0.70",
-    "@ai-sdk/google": "^4.0.59",
-    "@ai-sdk/mistral": "^4.0.37",
-    "@ai-sdk/openai": "^4.0.53",
-    "@ai-sdk/openai-compatible": "^3.0.41",
+    "@ai-sdk/cohere": "^4.0.37",
+    "@ai-sdk/deepseek": "^3.0.39",
+    "@ai-sdk/gateway": "^4.0.73",
+    "@ai-sdk/google": "^4.0.63",
+    "@ai-sdk/mistral": "^4.0.39",
+    "@ai-sdk/openai": "^4.0.57",
+    "@ai-sdk/openai-compatible": "^3.0.43",
     "@clack/prompts": "^1.7.0",
     "@lancedb/lancedb": "^0.38.0",
     "@larkup/core": "workspace:*",
     "@larkup/marketplace": "workspace:*",
     "@larkup/vector-stores": "workspace:*",
+    "@napi-rs/canvas": "1.0.8",
     "@pinecone-database/pinecone": "^8.1.0",
-    ai: "^7.0.87",
+    ai: "^7.0.91",
     "apache-arrow": "18.1.0",
     chalk: "^6.0.0",
     chromadb: "^3.5.0",
     "cli-table3": "^0.6.5",
     commander: "^15.0.0",
+    mammoth: "^1.12.0",
+    "pdf-parse": "2.4.5",
     zod: "^4.4.3"
   },
   devDependencies: {
@@ -4538,6 +4541,8 @@ function providerEnvironmentKey(provider) {
 // src/lib/local-files.ts
 import { promises as fs7 } from "fs";
 import path9 from "path";
+import mammoth from "mammoth";
+import { getData } from "pdf-parse/worker";
 var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
   ".txt",
   ".md",
@@ -4569,6 +4574,11 @@ var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
   ".sh",
   ".css"
 ]);
+var PDF_EXTENSIONS = /* @__PURE__ */ new Set([".pdf"]);
+var WORD_EXTENSIONS = /* @__PURE__ */ new Set([".docx"]);
+var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([".png", ".jpg", ".jpeg", ".webp", ".gif"]);
+var VIDEO_EXTENSIONS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".mkv"]);
+var AUDIO_EXTENSIONS = /* @__PURE__ */ new Set([".mp3", ".m4a", ".wav", ".ogg", ".flac"]);
 var MEDIA_EXTENSIONS = /* @__PURE__ */ new Set([
   ".png",
   ".jpg",
@@ -4586,9 +4596,52 @@ var MEDIA_EXTENSIONS = /* @__PURE__ */ new Set([
   ".flac"
 ]);
 var SKIPPED_DIRECTORIES = /* @__PURE__ */ new Set([".git", ".larkup", "node_modules", "dist", "build"]);
+var pdfParserPromise;
+function extensionOf(filePath) {
+  return path9.extname(filePath).toLowerCase();
+}
+function normalizedExtension(extension) {
+  const value = extension.trim().toLowerCase();
+  return value ? value.startsWith(".") ? value : `.${value}` : "";
+}
+async function getPdfParser() {
+  if (!pdfParserPromise) {
+    pdfParserPromise = (async () => {
+      const canvas = await import("@napi-rs/canvas");
+      Object.assign(globalThis, {
+        DOMMatrix: canvas.DOMMatrix,
+        ImageData: canvas.ImageData,
+        Path2D: canvas.Path2D
+      });
+      const { PDFParse } = await import("pdf-parse");
+      PDFParse.setWorker(getData());
+      return PDFParse;
+    })();
+  }
+  return pdfParserPromise;
+}
 function isMediaPath(filePath) {
   return MEDIA_EXTENSIONS.has(path9.extname(filePath).toLowerCase());
 }
+function filterLocalFiles(files, filter = {}) {
+  const requestedExtensions = new Set(
+    (filter.extensions ?? []).map(normalizedExtension).filter(Boolean)
+  );
+  const requestedKinds = new Set(filter.kinds ?? []);
+  if (requestedExtensions.size === 0 && requestedKinds.size === 0) return files;
+  return files.filter((filePath) => {
+    const extension = extensionOf(filePath);
+    if (requestedExtensions.has(extension)) return true;
+    if (requestedKinds.has("media") && MEDIA_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("image") && IMAGE_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("audio") && AUDIO_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("video") && VIDEO_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("pdf") && PDF_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("json") && (extension === ".json" || extension === ".jsonl"))
+      return true;
+    return requestedKinds.has("text") && TEXT_EXTENSIONS.has(extension);
+  });
+}
 async function collectFiles(inputs) {
   const files = [];
   for (const input of inputs) {
@@ -4617,8 +4670,22 @@ async function walk(directory, files) {
 async function readTextFiles(files) {
   const documents = [];
   for (const filePath of files) {
-    if (!TEXT_EXTENSIONS.has(path9.extname(filePath).toLowerCase())) continue;
-    const content = await fs7.readFile(filePath, "utf8");
+    const extension = extensionOf(filePath);
+    let content;
+    if (PDF_EXTENSIONS.has(extension)) {
+      const PDFParse = await getPdfParser();
+      const parser = new PDFParse({ data: new Uint8Array(await fs7.readFile(filePath)) });
+      try {
+        content = (await parser.getText()).text;
+      } finally {
+        await parser.destroy();
+      }
+    } else if (WORD_EXTENSIONS.has(extension)) {
+      content = (await mammoth.extractRawText({ buffer: await fs7.readFile(filePath) })).value;
+    } else {
+      if (!TEXT_EXTENSIONS.has(extension)) continue;
+      content = await fs7.readFile(filePath, "utf8");
+    }
     if (!content.trim() || content.includes("\0")) continue;
     documents.push({
       path: filePath,
@@ -4886,7 +4953,7 @@ var S3StorageProvider = class {
   }
   async client() {
     if (!this._client) {
-      const { S3Client } = await import("./dist-es-LRDDK5LI.js");
+      const { S3Client } = await import("./dist-es-3QFDPX44.js");
       this._client = new S3Client({
         region: this.region,
         ...this._credentials ? { credentials: this._credentials } : {}
@@ -4905,7 +4972,7 @@ var S3StorageProvider = class {
     return { bucket: withoutScheme.slice(0, slashIndex), key: withoutScheme.slice(slashIndex + 1) };
   }
   async store(key, data, mimeType) {
-    const { PutObjectCommand } = await import("./dist-es-LRDDK5LI.js");
+    const { PutObjectCommand } = await import("./dist-es-3QFDPX44.js");
     const client = await this.client();
     const objectKey = this.objectKey(key);
     await client.send(
@@ -4923,7 +4990,7 @@ var S3StorageProvider = class {
     return this.store(key, await fs9.readFile(sourcePath), mimeType);
   }
   async retrieve(uri) {
-    const { GetObjectCommand } = await import("./dist-es-LRDDK5LI.js");
+    const { GetObjectCommand } = await import("./dist-es-3QFDPX44.js");
     const { bucket, key } = this.parseUri(uri);
     const client = await this.client();
     const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
@@ -4939,8 +5006,8 @@ var S3StorageProvider = class {
     return void 0;
   }
   async getReadUrl(uri, expiresInSecs) {
-    const { GetObjectCommand } = await import("./dist-es-LRDDK5LI.js");
-    const { getSignedUrl } = await import("./dist-es-LWW2DNRF.js");
+    const { GetObjectCommand } = await import("./dist-es-3QFDPX44.js");
+    const { getSignedUrl } = await import("./dist-es-LH5Y4P7D.js");
     const { bucket, key } = this.parseUri(uri);
     const client = await this.client();
     return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), {
@@ -4948,7 +5015,7 @@ var S3StorageProvider = class {
     });
   }
   async delete(uri) {
-    const { DeleteObjectCommand } = await import("./dist-es-LRDDK5LI.js");
+    const { DeleteObjectCommand } = await import("./dist-es-3QFDPX44.js");
     const { bucket, key } = this.parseUri(uri);
     const client = await this.client();
     try {
@@ -5897,13 +5964,34 @@ function deriveCliAudioSignals(chunks) {
 }
 
 // src/commands/index-cmd.ts
+function selectedKinds(options) {
+  return [
+    ...options.pdf ? ["pdf"] : [],
+    ...options.json ? ["json"] : [],
+    ...options.media ? ["media"] : [],
+    ...options.image ? ["image"] : [],
+    ...options.audio ? ["audio"] : [],
+    ...options.video ? ["video"] : []
+  ];
+}
+function progressBar(current, total) {
+  if (total <= 0) return "Preparing chunks";
+  const width = 24;
+  const completed = Math.min(width, Math.round(current / total * width));
+  const percentage = Math.min(100, Math.round(current / total * 100));
+  return `[${"\u2588".repeat(completed)}${"\u2591".repeat(width - completed)}] ${percentage}%`;
+}
 async function indexCommand(inputs, options) {
   await inProjectScope(options.project, async () => {
     await requireActiveProject();
     if (inputs.length > 0) {
       const files = await collectFiles(inputs);
-      const textFiles = await readTextFiles(files);
-      const mediaFiles = files.filter(isMediaPath);
+      const selected = filterLocalFiles(files, {
+        kinds: selectedKinds(options),
+        extensions: options.extension?.split(",")
+      });
+      const textFiles = await readTextFiles(selected);
+      const mediaFiles = selected.filter(isMediaPath);
       for (const document of textFiles) {
         await addDocument({ title: document.title, content: document.content, source: "files" });
       }
@@ -5939,7 +6027,9 @@ async function indexCommand(inputs, options) {
       const r = await readRun();
       if (!r || done) return;
       if (r.totalChunks > 0) {
-        s.message(`${r.status.padEnd(10)} ${r.processedChunks}/${r.totalChunks} chunks`);
+        s.message(
+          `${r.status.padEnd(10)} ${progressBar(r.processedChunks, r.totalChunks)} ${r.processedChunks}/${r.totalChunks} chunks`
+        );
       } else {
         s.message(`${r.status}\u2026`);
       }
@@ -6218,8 +6308,8 @@ async function chatCommand(options) {
     const provider = config.chatProvider || config.embeddingProvider;
     const chatModels = (await getAllModels()).filter((model) => model.type === "language").map(toChatDescriptor);
     const chatModelId = options.model || config.chatModelId || getDefaultChatModel(chatModels, provider)?.id || "openai/gpt-4o-mini";
-    const modelProvider = chatModelId.startsWith("custom:") ? "custom" : chatModelId.includes("/") ? chatModelId.split("/", 1)[0] : provider;
-    if (provider !== "vercel_ai_gateway" && modelProvider !== provider) {
+    const modelProvider2 = chatModelId.startsWith("custom:") ? "custom" : chatModelId.includes("/") ? chatModelId.split("/", 1)[0] : provider;
+    if (provider !== "vercel_ai_gateway" && modelProvider2 !== provider) {
       throw new Error(
         `Model "${chatModelId}" is not available through the configured ${provider} provider. Choose a matching model or change the chat provider with \`larkup settings\`.`
       );
@@ -6954,6 +7044,101 @@ async function installCommand(options) {
   if (notEntitled.length) log.error(`Not entitled for this organization: ${notEntitled.join(", ")}`);
 }
 
+// src/commands/ai-model.ts
+function modelProvider(modelId) {
+  return modelId.split("/")[0] || "openai";
+}
+function supportsVision(model) {
+  return model.type === "language" && Boolean(model.tags?.includes("vision"));
+}
+function printModel(model) {
+  log.info(`${model.id}${model.name && model.name !== model.id ? `  ${model.name}` : ""}`);
+  if (model.description) log.dim(`  ${model.description}`);
+}
+async function listModels(type) {
+  const models = await getAllModels();
+  const selectedType = type ?? "embedding";
+  if (selectedType === "embedding") {
+    for (const model of EMBEDDING_MODELS) printModel({ ...model, name: model.label });
+    return;
+  }
+  const matching = models.filter(
+    selectedType === "vision" ? supportsVision : (model) => model.type === "language" && model.tags?.includes("tool-use")
+  );
+  for (const model of matching) printModel(model);
+}
+function validateLanguageModel(models, modelId, capability) {
+  const model = models.find((candidate) => candidate.id === modelId);
+  if (!model || model.type !== "language") {
+    throw new Error(
+      `Unknown ${capability} model "${modelId}". Run: larkup ai-model list --type ${capability}`
+    );
+  }
+  if (capability === "vision" && !supportsVision(model)) {
+    throw new Error(
+      `Model "${modelId}" is not vision-capable. Run: larkup ai-model list --type vision`
+    );
+  }
+  return model;
+}
+async function aiModelCommand(action, options) {
+  await inProjectScope(options.project, async () => {
+    await requireActiveProject();
+    if (action === "list") {
+      await listModels(options.type);
+      return;
+    }
+    if (action && action !== "show") {
+      throw new Error(
+        `Unknown ai-model action "${action}". Use "list" or omit the action to configure models.`
+      );
+    }
+    const config = await readConfig();
+    const apiKey = options.apiKey ?? options.apikey;
+    if (!options.embedding && !options.chat && !options.vision) {
+      if (apiKey) {
+        await writeConfig({ ...config, embeddingApiKey: apiKey });
+        log.success("Embedding API key saved for the active Project.");
+        return;
+      }
+      log.info(`Embedding  ${config.embeddingModelId}`);
+      log.info(`Chat       ${config.chatModelId ?? "not configured"}`);
+      log.info(`Vision     ${config.visionModelId ?? "not configured"}`);
+      log.dim(
+        "Set one with: larkup ai-model --embedding openai/text-embedding-3-small --api-key <key>"
+      );
+      return;
+    }
+    const models = await getAllModels();
+    const next = { ...config };
+    if (options.embedding) {
+      const embedding = getEmbeddingModel(options.embedding);
+      if (!embedding) {
+        throw new Error(
+          `Unknown embedding model "${options.embedding}". Run: larkup ai-model list --type embedding`
+        );
+      }
+      next.embeddingModelId = embedding.id;
+      next.embeddingProvider = embedding.provider;
+      if (apiKey) next.embeddingApiKey = apiKey;
+    }
+    if (options.chat) {
+      const chat = validateLanguageModel(models, options.chat, "chat");
+      next.chatModelId = chat.id;
+      next.chatProvider = modelProvider(chat.id);
+      if (apiKey) next.chatApiKey = apiKey;
+    }
+    if (options.vision) {
+      const vision = validateLanguageModel(models, options.vision, "vision");
+      next.visionModelId = vision.id;
+      next.visionProvider = modelProvider(vision.id);
+      if (apiKey) next.visionApiKey = apiKey;
+    }
+    await writeConfig(next);
+    log.success("AI model settings saved. The Web UI now uses the same Project configuration.");
+  });
+}
+
 // src/index.ts
 var program = new Command();
 program.name("larkup").description("Build, index, and serve a RAG pipeline from the terminal.").version(package_default.version);
@@ -6985,9 +7170,26 @@ program.command("add-doc").description("Add a document to the corpus").option("-
 program.command("documents [action] [idOrPath]").alias("docs").description("List, show, update, remove, or export corpus documents").option("--title <title>", "Set a document title when updating").option("--text <content>", "Set document content when updating").option("--file <path>", "Read updated document content from a file").option("--url <url>", "Set the source URL when updating").option("--format <csv|jsonl>", "Corpus export format", "csv").option("--out <path>", "Corpus export destination").option("--project <id>", "Target a specific Project instead of the active one").action(async (action, idOrPath, options) => {
   await documentsCommand(action, idOrPath, options);
 });
-program.command("index [sources...]").description("Load files or folders, then chunk \u2192 embed \u2192 store the corpus").option("--project <id>", "Target a specific Project instead of the active one").option("--no-run", "Load supplied sources without building the index").option("--incremental", "Index only documents added since the last completed run").action(async (sources, options) => {
+program.command("index [sources...]").description("Load files or folders, then chunk \u2192 embed \u2192 store the corpus").option("--project <id>", "Target a specific Project instead of the active one").option("--no-run", "Load supplied sources without building the index").option("--incremental", "Index only documents added since the last completed run").option("--pdf", "Index PDF files only").option("--json", "Index JSON and JSONL files only").option("--media", "Index images, audio, and video only").option("--image", "Index image files only").option("--audio", "Index audio files only").option("--video", "Index video files only").option("--extension <extensions>", "Comma-separated extensions, for example .json,.pdf").action(async (sources, options) => {
+  await indexCommand(sources ?? [], options);
+});
+program.command("indexing [sources...]").description("Alias for index").option("--project <id>", "Target a specific Project instead of the active one").option("--no-run", "Load supplied sources without building the index").option("--incremental", "Index only documents added since the last completed run").option("--pdf", "Index PDF files only").option("--json", "Index JSON and JSONL files only").option("--media", "Index images, audio, and video only").option("--image", "Index image files only").option("--audio", "Index audio files only").option("--video", "Index video files only").option("--extension <extensions>", "Comma-separated extensions, for example .json,.pdf").action(async (sources, options) => {
   await indexCommand(sources ?? [], options);
 });
+function configureAiModelCommand(command) {
+  return command.option("--embedding <model>", "Set the embedding model ID").option("--chat <model>", "Set the chat model ID").option("--vision <model>", "Set the vision model ID").option("--api-key <key>", "Set the API key for the selected model type(s)").option("--apikey <key>", "Alias for --api-key").option("--project <id>", "Target a specific Project instead of the active one").option("--type <embedding|chat|vision>", "Filter model list by capability");
+}
+configureAiModelCommand(
+  program.command("ai-model [action]").description("List or configure embedding, chat, and vision AI models")
+).action(async (action, options) => {
+  await aiModelCommand(action, options);
+});
+var setCommand = program.command("set").description("Set Project configuration values");
+configureAiModelCommand(
+  setCommand.command("ai-model").description("Configure AI models (alias for ai-model)")
+).action(async (options) => {
+  await aiModelCommand(void 0, options);
+});
 program.command("media [sources...]").description("Add image, audio, or video files and stream processing progress").option("--no-index", "Process media without rebuilding the index").option("--project <id>", "Target a specific Project instead of the active one").action(async (sources, options) => {
   await mediaCommand(sources ?? [], options);
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larkup/cli",
-  "version": "0.2.31",
+  "version": "0.3.0",
   "engines": {
     "node": ">=22"
   },
@@ -15,26 +15,29 @@
     "dist"
   ],
   "dependencies": {
-    "@ai-sdk/cohere": "^4.0.35",
-    "@ai-sdk/deepseek": "^3.0.37",
-    "@ai-sdk/gateway": "^4.0.70",
-    "@ai-sdk/google": "^4.0.59",
-    "@ai-sdk/mistral": "^4.0.37",
-    "@ai-sdk/openai": "^4.0.53",
-    "@ai-sdk/openai-compatible": "^3.0.41",
+    "@ai-sdk/cohere": "^4.0.37",
+    "@ai-sdk/deepseek": "^3.0.39",
+    "@ai-sdk/gateway": "^4.0.73",
+    "@ai-sdk/google": "^4.0.63",
+    "@ai-sdk/mistral": "^4.0.39",
+    "@ai-sdk/openai": "^4.0.57",
+    "@ai-sdk/openai-compatible": "^3.0.43",
     "@clack/prompts": "^1.7.0",
     "@lancedb/lancedb": "^0.38.0",
+    "@napi-rs/canvas": "1.0.8",
     "@pinecone-database/pinecone": "^8.1.0",
-    "ai": "^7.0.87",
+    "ai": "^7.0.91",
     "apache-arrow": "18.1.0",
     "chalk": "^6.0.0",
     "chromadb": "^3.5.0",
     "cli-table3": "^0.6.5",
     "commander": "^15.0.0",
+    "mammoth": "^1.12.0",
+    "pdf-parse": "2.4.5",
     "zod": "^4.4.3",
-    "@larkup/core": "0.5.3",
-    "@larkup/marketplace": "0.2.3",
-    "@larkup/vector-stores": "0.1.27"
+    "@larkup/core": "0.5.4",
+    "@larkup/vector-stores": "0.1.27",
+    "@larkup/marketplace": "0.2.4"
   },
   "devDependencies": {
     "tsup": "^8.5.1",