@larkup/cli 0.2.32 → 0.3.0

package/README.md

@@ -12,11 +12,20 @@ npm install -g @larkup/cli
 
 ```bash
 larkup init product-docs
+larkup ai-model --embedding openai/text-embedding-3-small --api-key "$OPENAI_API_KEY"
 larkup index ./knowledge
 larkup dev
 ```
 
-`index` accepts individual files or folders, including text, source code, CSV, JSON, PDF, and Word documents. It streams loading and indexing progress.
+`index` accepts individual files or folders, including text, source code, CSV, JSON, PDF, and Word documents. It streams loading and indexing progress, and writes the same Project state shown in the Web UI.
+
+```bash
+larkup index ./folder --pdf
+larkup index ./folder --media
+larkup index ./folder --video
+larkup index ./folder --extension .json,.pdf
+larkup ai-model list --type embedding
+```
 
 Use the dedicated media command for images, audio, and video:
 
@@ -28,6 +37,7 @@ larkup media ./demo.mp4
 ## Commands
 
 - `larkup index [sources...]` loads sources and builds the index.
+- `larkup ai-model [list]` lists or configures embedding, chat, and vision models.
 - `larkup dev [name]` creates a workspace (default: `my-larkup`) and runs its local server.
 - `larkup documents` manages and exports corpus documents.
 - `larkup media [sources...]` processes image, audio, and video files.

package/dist/index.js

@@ -36,7 +36,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "@larkup/cli",
-  version: "0.2.32",
+  version: "0.3.0",
   engines: {
     node: ">=22"
   },
@@ -69,6 +69,7 @@ var package_default = {
     "@larkup/core": "workspace:*",
     "@larkup/marketplace": "workspace:*",
     "@larkup/vector-stores": "workspace:*",
+    "@napi-rs/canvas": "1.0.8",
     "@pinecone-database/pinecone": "^8.1.0",
     ai: "^7.0.91",
     "apache-arrow": "18.1.0",
@@ -76,6 +77,8 @@ var package_default = {
     chromadb: "^3.5.0",
     "cli-table3": "^0.6.5",
     commander: "^15.0.0",
+    mammoth: "^1.12.0",
+    "pdf-parse": "2.4.5",
     zod: "^4.4.3"
   },
   devDependencies: {
@@ -4538,6 +4541,8 @@ function providerEnvironmentKey(provider) {
 // src/lib/local-files.ts
 import { promises as fs7 } from "fs";
 import path9 from "path";
+import mammoth from "mammoth";
+import { getData } from "pdf-parse/worker";
 var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
   ".txt",
   ".md",
@@ -4569,6 +4574,11 @@ var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
   ".sh",
   ".css"
 ]);
+var PDF_EXTENSIONS = /* @__PURE__ */ new Set([".pdf"]);
+var WORD_EXTENSIONS = /* @__PURE__ */ new Set([".docx"]);
+var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([".png", ".jpg", ".jpeg", ".webp", ".gif"]);
+var VIDEO_EXTENSIONS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".mkv"]);
+var AUDIO_EXTENSIONS = /* @__PURE__ */ new Set([".mp3", ".m4a", ".wav", ".ogg", ".flac"]);
 var MEDIA_EXTENSIONS = /* @__PURE__ */ new Set([
   ".png",
   ".jpg",
@@ -4586,9 +4596,52 @@ var MEDIA_EXTENSIONS = /* @__PURE__ */ new Set([
   ".flac"
 ]);
 var SKIPPED_DIRECTORIES = /* @__PURE__ */ new Set([".git", ".larkup", "node_modules", "dist", "build"]);
+var pdfParserPromise;
+function extensionOf(filePath) {
+  return path9.extname(filePath).toLowerCase();
+}
+function normalizedExtension(extension) {
+  const value = extension.trim().toLowerCase();
+  return value ? value.startsWith(".") ? value : `.${value}` : "";
+}
+async function getPdfParser() {
+  if (!pdfParserPromise) {
+    pdfParserPromise = (async () => {
+      const canvas = await import("@napi-rs/canvas");
+      Object.assign(globalThis, {
+        DOMMatrix: canvas.DOMMatrix,
+        ImageData: canvas.ImageData,
+        Path2D: canvas.Path2D
+      });
+      const { PDFParse } = await import("pdf-parse");
+      PDFParse.setWorker(getData());
+      return PDFParse;
+    })();
+  }
+  return pdfParserPromise;
+}
 function isMediaPath(filePath) {
   return MEDIA_EXTENSIONS.has(path9.extname(filePath).toLowerCase());
 }
+function filterLocalFiles(files, filter = {}) {
+  const requestedExtensions = new Set(
+    (filter.extensions ?? []).map(normalizedExtension).filter(Boolean)
+  );
+  const requestedKinds = new Set(filter.kinds ?? []);
+  if (requestedExtensions.size === 0 && requestedKinds.size === 0) return files;
+  return files.filter((filePath) => {
+    const extension = extensionOf(filePath);
+    if (requestedExtensions.has(extension)) return true;
+    if (requestedKinds.has("media") && MEDIA_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("image") && IMAGE_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("audio") && AUDIO_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("video") && VIDEO_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("pdf") && PDF_EXTENSIONS.has(extension)) return true;
+    if (requestedKinds.has("json") && (extension === ".json" || extension === ".jsonl"))
+      return true;
+    return requestedKinds.has("text") && TEXT_EXTENSIONS.has(extension);
+  });
+}
 async function collectFiles(inputs) {
   const files = [];
   for (const input of inputs) {
@@ -4617,8 +4670,22 @@ async function walk(directory, files) {
 async function readTextFiles(files) {
   const documents = [];
   for (const filePath of files) {
-    if (!TEXT_EXTENSIONS.has(path9.extname(filePath).toLowerCase())) continue;
-    const content = await fs7.readFile(filePath, "utf8");
+    const extension = extensionOf(filePath);
+    let content;
+    if (PDF_EXTENSIONS.has(extension)) {
+      const PDFParse = await getPdfParser();
+      const parser = new PDFParse({ data: new Uint8Array(await fs7.readFile(filePath)) });
+      try {
+        content = (await parser.getText()).text;
+      } finally {
+        await parser.destroy();
+      }
+    } else if (WORD_EXTENSIONS.has(extension)) {
+      content = (await mammoth.extractRawText({ buffer: await fs7.readFile(filePath) })).value;
+    } else {
+      if (!TEXT_EXTENSIONS.has(extension)) continue;
+      content = await fs7.readFile(filePath, "utf8");
+    }
     if (!content.trim() || content.includes("\0")) continue;
     documents.push({
       path: filePath,
@@ -5897,13 +5964,34 @@ function deriveCliAudioSignals(chunks) {
 }
 
 // src/commands/index-cmd.ts
+function selectedKinds(options) {
+  return [
+    ...options.pdf ? ["pdf"] : [],
+    ...options.json ? ["json"] : [],
+    ...options.media ? ["media"] : [],
+    ...options.image ? ["image"] : [],
+    ...options.audio ? ["audio"] : [],
+    ...options.video ? ["video"] : []
+  ];
+}
+function progressBar(current, total) {
+  if (total <= 0) return "Preparing chunks";
+  const width = 24;
+  const completed = Math.min(width, Math.round(current / total * width));
+  const percentage = Math.min(100, Math.round(current / total * 100));
+  return `[${"\u2588".repeat(completed)}${"\u2591".repeat(width - completed)}] ${percentage}%`;
+}
 async function indexCommand(inputs, options) {
   await inProjectScope(options.project, async () => {
     await requireActiveProject();
     if (inputs.length > 0) {
       const files = await collectFiles(inputs);
-      const textFiles = await readTextFiles(files);
-      const mediaFiles = files.filter(isMediaPath);
+      const selected = filterLocalFiles(files, {
+        kinds: selectedKinds(options),
+        extensions: options.extension?.split(",")
+      });
+      const textFiles = await readTextFiles(selected);
+      const mediaFiles = selected.filter(isMediaPath);
       for (const document of textFiles) {
         await addDocument({ title: document.title, content: document.content, source: "files" });
       }
@@ -5939,7 +6027,9 @@ async function indexCommand(inputs, options) {
       const r = await readRun();
       if (!r || done) return;
       if (r.totalChunks > 0) {
-        s.message(`${r.status.padEnd(10)} ${r.processedChunks}/${r.totalChunks} chunks`);
+        s.message(
+          `${r.status.padEnd(10)} ${progressBar(r.processedChunks, r.totalChunks)} ${r.processedChunks}/${r.totalChunks} chunks`
+        );
       } else {
         s.message(`${r.status}\u2026`);
       }
@@ -6218,8 +6308,8 @@ async function chatCommand(options) {
     const provider = config.chatProvider || config.embeddingProvider;
     const chatModels = (await getAllModels()).filter((model) => model.type === "language").map(toChatDescriptor);
     const chatModelId = options.model || config.chatModelId || getDefaultChatModel(chatModels, provider)?.id || "openai/gpt-4o-mini";
-    const modelProvider = chatModelId.startsWith("custom:") ? "custom" : chatModelId.includes("/") ? chatModelId.split("/", 1)[0] : provider;
-    if (provider !== "vercel_ai_gateway" && modelProvider !== provider) {
+    const modelProvider2 = chatModelId.startsWith("custom:") ? "custom" : chatModelId.includes("/") ? chatModelId.split("/", 1)[0] : provider;
+    if (provider !== "vercel_ai_gateway" && modelProvider2 !== provider) {
       throw new Error(
         `Model "${chatModelId}" is not available through the configured ${provider} provider. Choose a matching model or change the chat provider with \`larkup settings\`.`
       );
@@ -6954,6 +7044,101 @@ async function installCommand(options) {
   if (notEntitled.length) log.error(`Not entitled for this organization: ${notEntitled.join(", ")}`);
 }
 
+// src/commands/ai-model.ts
+function modelProvider(modelId) {
+  return modelId.split("/")[0] || "openai";
+}
+function supportsVision(model) {
+  return model.type === "language" && Boolean(model.tags?.includes("vision"));
+}
+function printModel(model) {
+  log.info(`${model.id}${model.name && model.name !== model.id ? `  ${model.name}` : ""}`);
+  if (model.description) log.dim(`  ${model.description}`);
+}
+async function listModels(type) {
+  const models = await getAllModels();
+  const selectedType = type ?? "embedding";
+  if (selectedType === "embedding") {
+    for (const model of EMBEDDING_MODELS) printModel({ ...model, name: model.label });
+    return;
+  }
+  const matching = models.filter(
+    selectedType === "vision" ? supportsVision : (model) => model.type === "language" && model.tags?.includes("tool-use")
+  );
+  for (const model of matching) printModel(model);
+}
+function validateLanguageModel(models, modelId, capability) {
+  const model = models.find((candidate) => candidate.id === modelId);
+  if (!model || model.type !== "language") {
+    throw new Error(
+      `Unknown ${capability} model "${modelId}". Run: larkup ai-model list --type ${capability}`
+    );
+  }
+  if (capability === "vision" && !supportsVision(model)) {
+    throw new Error(
+      `Model "${modelId}" is not vision-capable. Run: larkup ai-model list --type vision`
+    );
+  }
+  return model;
+}
+async function aiModelCommand(action, options) {
+  await inProjectScope(options.project, async () => {
+    await requireActiveProject();
+    if (action === "list") {
+      await listModels(options.type);
+      return;
+    }
+    if (action && action !== "show") {
+      throw new Error(
+        `Unknown ai-model action "${action}". Use "list" or omit the action to configure models.`
+      );
+    }
+    const config = await readConfig();
+    const apiKey = options.apiKey ?? options.apikey;
+    if (!options.embedding && !options.chat && !options.vision) {
+      if (apiKey) {
+        await writeConfig({ ...config, embeddingApiKey: apiKey });
+        log.success("Embedding API key saved for the active Project.");
+        return;
+      }
+      log.info(`Embedding  ${config.embeddingModelId}`);
+      log.info(`Chat       ${config.chatModelId ?? "not configured"}`);
+      log.info(`Vision     ${config.visionModelId ?? "not configured"}`);
+      log.dim(
+        "Set one with: larkup ai-model --embedding openai/text-embedding-3-small --api-key <key>"
+      );
+      return;
+    }
+    const models = await getAllModels();
+    const next = { ...config };
+    if (options.embedding) {
+      const embedding = getEmbeddingModel(options.embedding);
+      if (!embedding) {
+        throw new Error(
+          `Unknown embedding model "${options.embedding}". Run: larkup ai-model list --type embedding`
+        );
+      }
+      next.embeddingModelId = embedding.id;
+      next.embeddingProvider = embedding.provider;
+      if (apiKey) next.embeddingApiKey = apiKey;
+    }
+    if (options.chat) {
+      const chat = validateLanguageModel(models, options.chat, "chat");
+      next.chatModelId = chat.id;
+      next.chatProvider = modelProvider(chat.id);
+      if (apiKey) next.chatApiKey = apiKey;
+    }
+    if (options.vision) {
+      const vision = validateLanguageModel(models, options.vision, "vision");
+      next.visionModelId = vision.id;
+      next.visionProvider = modelProvider(vision.id);
+      if (apiKey) next.visionApiKey = apiKey;
+    }
+    await writeConfig(next);
+    log.success("AI model settings saved. The Web UI now uses the same Project configuration.");
+  });
+}
+
 // src/index.ts
 var program = new Command();
 program.name("larkup").description("Build, index, and serve a RAG pipeline from the terminal.").version(package_default.version);
@@ -6985,9 +7170,26 @@ program.command("add-doc").description("Add a document to the corpus").option("-
 program.command("documents [action] [idOrPath]").alias("docs").description("List, show, update, remove, or export corpus documents").option("--title <title>", "Set a document title when updating").option("--text <content>", "Set document content when updating").option("--file <path>", "Read updated document content from a file").option("--url <url>", "Set the source URL when updating").option("--format <csv|jsonl>", "Corpus export format", "csv").option("--out <path>", "Corpus export destination").option("--project <id>", "Target a specific Project instead of the active one").action(async (action, idOrPath, options) => {
   await documentsCommand(action, idOrPath, options);
 });
-program.command("index [sources...]").description("Load files or folders, then chunk \u2192 embed \u2192 store the corpus").option("--project <id>", "Target a specific Project instead of the active one").option("--no-run", "Load supplied sources without building the index").option("--incremental", "Index only documents added since the last completed run").action(async (sources, options) => {
+program.command("index [sources...]").description("Load files or folders, then chunk \u2192 embed \u2192 store the corpus").option("--project <id>", "Target a specific Project instead of the active one").option("--no-run", "Load supplied sources without building the index").option("--incremental", "Index only documents added since the last completed run").option("--pdf", "Index PDF files only").option("--json", "Index JSON and JSONL files only").option("--media", "Index images, audio, and video only").option("--image", "Index image files only").option("--audio", "Index audio files only").option("--video", "Index video files only").option("--extension <extensions>", "Comma-separated extensions, for example .json,.pdf").action(async (sources, options) => {
+  await indexCommand(sources ?? [], options);
+});
+program.command("indexing [sources...]").description("Alias for index").option("--project <id>", "Target a specific Project instead of the active one").option("--no-run", "Load supplied sources without building the index").option("--incremental", "Index only documents added since the last completed run").option("--pdf", "Index PDF files only").option("--json", "Index JSON and JSONL files only").option("--media", "Index images, audio, and video only").option("--image", "Index image files only").option("--audio", "Index audio files only").option("--video", "Index video files only").option("--extension <extensions>", "Comma-separated extensions, for example .json,.pdf").action(async (sources, options) => {
   await indexCommand(sources ?? [], options);
 });
+function configureAiModelCommand(command) {
+  return command.option("--embedding <model>", "Set the embedding model ID").option("--chat <model>", "Set the chat model ID").option("--vision <model>", "Set the vision model ID").option("--api-key <key>", "Set the API key for the selected model type(s)").option("--apikey <key>", "Alias for --api-key").option("--project <id>", "Target a specific Project instead of the active one").option("--type <embedding|chat|vision>", "Filter model list by capability");
+}
+configureAiModelCommand(
+  program.command("ai-model [action]").description("List or configure embedding, chat, and vision AI models")
+).action(async (action, options) => {
+  await aiModelCommand(action, options);
+});
+var setCommand = program.command("set").description("Set Project configuration values");
+configureAiModelCommand(
+  setCommand.command("ai-model").description("Configure AI models (alias for ai-model)")
+).action(async (options) => {
+  await aiModelCommand(void 0, options);
+});
 program.command("media [sources...]").description("Add image, audio, or video files and stream processing progress").option("--no-index", "Process media without rebuilding the index").option("--project <id>", "Target a specific Project instead of the active one").action(async (sources, options) => {
   await mediaCommand(sources ?? [], options);
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larkup/cli",
-  "version": "0.2.32",
+  "version": "0.3.0",
   "engines": {
     "node": ">=22"
   },
@@ -24,6 +24,7 @@
     "@ai-sdk/openai-compatible": "^3.0.43",
     "@clack/prompts": "^1.7.0",
     "@lancedb/lancedb": "^0.38.0",
+    "@napi-rs/canvas": "1.0.8",
     "@pinecone-database/pinecone": "^8.1.0",
     "ai": "^7.0.91",
     "apache-arrow": "18.1.0",
@@ -31,10 +32,12 @@
     "chromadb": "^3.5.0",
     "cli-table3": "^0.6.5",
     "commander": "^15.0.0",
+    "mammoth": "^1.12.0",
+    "pdf-parse": "2.4.5",
     "zod": "^4.4.3",
-    "@larkup/marketplace": "0.2.4",
     "@larkup/core": "0.5.4",
-    "@larkup/vector-stores": "0.1.27"
+    "@larkup/vector-stores": "0.1.27",
+    "@larkup/marketplace": "0.2.4"
   },
   "devDependencies": {
     "tsup": "^8.5.1",