@larkup/cli 0.2.16 → 0.2.17

package/dist/{chunk-UCYL66UI.js → chunk-7HVIRRTX.js}

@@ -90,13 +90,12 @@ var FALLBACK_REGISTRY = {
     name: "Video & Audio",
     description: "Index video and audio files with transcription and frame analysis.",
     category: "media",
-    version: "0.2.1",
+    version: "0.3.1",
     pricing: "free",
     emoji: "\u{1F3AC}",
     icon: "Film",
     packageName: "@larkup/tool-video-audio",
     installSize: "~15 MB",
-    systemDeps: ["ffmpeg"],
     author: "Larkup",
     capabilities: [
       "video-indexing",
@@ -136,7 +135,7 @@ var FALLBACK_REGISTRY = {
         label: "Transcription language",
         type: "text",
         defaultValue: "auto",
-        help: "Use auto to infer Arabic from the media title and use provider detection otherwise. Enter a language code such as ar, ar-EG, en, or de to override it."
+        help: "Use auto for provider language detection, or enter a language code such as ar, ar-EG, en, or de to override it."
       },
       {
         key: "audioApiKey",

package/dist/{chunk-EVFK76V4.js → chunk-STUKQORW.js}

@@ -1,7 +1,7 @@
 import {
   getToolById,
   invalidateRegistryCache
-} from "./chunk-UCYL66UI.js";
+} from "./chunk-7HVIRRTX.js";
 import {
   __require
 } from "./chunk-3RG5ZIWI.js";
@@ -414,7 +414,7 @@ async function uninstallTool(toolId) {
     }
   }
   invalidateRegistryCache();
-  const { unloadTool: unloadTool2 } = await import("./tool-loader-AM6O3TCX.js");
+  const { unloadTool: unloadTool2 } = await import("./tool-loader-K33ISDZP.js");
   unloadTool2(toolId);
 }
 async function notifyHubInstall(toolId) {
@@ -485,7 +485,7 @@ function unloadTool(toolId) {
   moduleCache.delete(toolId);
 }
 async function hasCapability(capability) {
-  const { getToolsWithCapability } = await import("./tool-registry-ED45DB5F.js");
+  const { getToolsWithCapability } = await import("./tool-registry-XQVQ6KQF.js");
   const tools = await getToolsWithCapability(capability);
   for (const t of tools) {
     const installed = await getInstalledTool(t.id);

package/dist/index.js

@@ -19,11 +19,11 @@ import {
   isToolInstalled,
   loadTool,
   uninstallTool
-} from "./chunk-EVFK76V4.js";
+} from "./chunk-STUKQORW.js";
 import {
   getAllTools,
   getToolById
-} from "./chunk-UCYL66UI.js";
+} from "./chunk-7HVIRRTX.js";
 import {
   __require
 } from "./chunk-3RG5ZIWI.js";
@@ -34,7 +34,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "@larkup/cli",
-  version: "0.2.16",
+  version: "0.2.17",
   publishConfig: {
     access: "public"
   },
@@ -4341,6 +4341,24 @@ async function createAdapter(config, onRateLimit) {
 
 // ../../packages/core/src/indexing/indexer.ts
 var EMBED_BATCH = 64;
+var MAX_RATE_LIMIT_RETRIES = 5;
+function isRateLimitError(error) {
+  const candidate = error;
+  const message = String(candidate?.message ?? error ?? "").toLowerCase();
+  return candidate?.status === 429 || candidate?.statusCode === 429 || /\b429\b|rate[ -]?limit|too many requests|resource_exhausted|quota exceeded|quota.*exceed/.test(
+    message
+  );
+}
+function rateLimitDelayMs(error, attempt) {
+  const message = String(error?.message ?? error ?? "");
+  const retryMatch = message.match(
+    /(?:retry(?:\s+after|\s+in)?|try again(?:\s+in)?)\s*(\d+(?:\.\d+)?)\s*(?:s|sec(?:ond)?s?)/i
+  );
+  if (retryMatch) {
+    return Math.min(12e4, Math.max(2e3, Math.ceil(Number(retryMatch[1]) * 1e3)));
+  }
+  return Math.min(6e4, 5e3 * 2 ** (attempt - 1));
+}
 async function createRun(config) {
   const now = (/* @__PURE__ */ new Date()).toISOString();
   const run = {
@@ -4367,7 +4385,7 @@ async function runIndexer(runId, config, previousRun = null) {
   };
   try {
     let docs = await readDocuments();
-    if (previousRun && previousRun.status === "completed") {
+    if (previousRun) {
       docs = docs.filter((document) => document.status !== "indexed");
     }
     if (docs.length === 0) {
@@ -4406,6 +4424,12 @@ async function runIndexer(runId, config, previousRun = null) {
       return;
     }
     await patchRun2({ totalChunks: chunks.length, status: "embedding" });
+    const chunksPerDocument = /* @__PURE__ */ new Map();
+    const processedPerDocument = /* @__PURE__ */ new Map();
+    const indexedDocumentIds = /* @__PURE__ */ new Set();
+    for (const chunk of chunks) {
+      chunksPerDocument.set(chunk.documentId, (chunksPerDocument.get(chunk.documentId) ?? 0) + 1);
+    }
     const adapter = await createAdapter(config, async (waitSecs, attempt) => {
       await patchRun2({
         warning: `Sparse model rate-limited \u2014 pausing ${waitSecs}s (retry ${attempt}/${3})\u2026`
@@ -4434,13 +4458,13 @@ async function runIndexer(runId, config, previousRun = null) {
           if (dim) batchDimensions = dim;
           break;
         } catch (err) {
-          const is429 = err?.status === 429 || err?.statusCode === 429 || String(err?.message ?? "").includes("429") || String(err?.message ?? "").includes("rate limit") || String(err?.message ?? "").toLowerCase().includes("too many requests");
-          if (is429 && attempt <= 3) {
-            const waitSecs = attempt * 10;
+          if (isRateLimitError(err) && attempt <= MAX_RATE_LIMIT_RETRIES) {
+            const waitMs = rateLimitDelayMs(err, attempt);
+            const waitSecs = Math.ceil(waitMs / 1e3);
             await patchRun2({
-              warning: `Dense model rate-limited \u2014 pausing ${waitSecs}s (retry ${attempt}/${3})\u2026`
+              warning: `Embedding provider quota reached \u2014 pausing ${waitSecs}s (retry ${attempt}/${MAX_RATE_LIMIT_RETRIES})\u2026`
             });
-            await new Promise((r) => setTimeout(r, waitSecs * 1e3));
+            await new Promise((r) => setTimeout(r, waitMs));
             currentDelayMs = Math.max(currentDelayMs, 2e3);
             attempt++;
           } else {
@@ -4453,6 +4477,10 @@ async function runIndexer(runId, config, previousRun = null) {
         await adapter.init(dimensions);
         if (!previousRun) {
           await adapter.reset();
+          await updateDocumentsStatus(
+            docs.map((document) => document.id),
+            "unindexed"
+          );
         }
         initialized = true;
         await patchRun2({ dimensions });
@@ -4475,13 +4503,13 @@ async function runIndexer(runId, config, previousRun = null) {
           await adapter.upsert(records);
           break;
         } catch (err) {
-          const is429 = err?.status === 429 || err?.statusCode === 429 || String(err?.message ?? "").includes("429") || String(err?.message ?? "").includes("rate limit") || String(err?.message ?? "").toLowerCase().includes("too many requests") || String(err?.message ?? "").includes("RESOURCE_EXHAUSTED");
-          if (is429 && upsertAttempt <= 3) {
-            const waitSecs = upsertAttempt * 10;
+          if (isRateLimitError(err) && upsertAttempt <= MAX_RATE_LIMIT_RETRIES) {
+            const waitMs = rateLimitDelayMs(err, upsertAttempt);
+            const waitSecs = Math.ceil(waitMs / 1e3);
             await patchRun2({
-              warning: `Vector store rate-limited \u2014 pausing ${waitSecs}s (retry ${upsertAttempt}/${3})\u2026`
+              warning: `Vector store rate-limited \u2014 pausing ${waitSecs}s (retry ${upsertAttempt}/${MAX_RATE_LIMIT_RETRIES})\u2026`
             });
-            await new Promise((r) => setTimeout(r, waitSecs * 1e3));
+            await new Promise((r) => setTimeout(r, waitMs));
             currentDelayMs = Math.max(currentDelayMs, 2e3);
             upsertAttempt++;
           } else {
@@ -4490,7 +4518,24 @@ async function runIndexer(runId, config, previousRun = null) {
         }
       }
       processed += batch.length;
-      await patchRun2({ processedChunks: processed, status: "embedding", warning: void 0 });
+      const completedDocumentIds = [];
+      for (const chunk of batch) {
+        const completed = (processedPerDocument.get(chunk.documentId) ?? 0) + 1;
+        processedPerDocument.set(chunk.documentId, completed);
+        if (completed === chunksPerDocument.get(chunk.documentId) && !indexedDocumentIds.has(chunk.documentId)) {
+          indexedDocumentIds.add(chunk.documentId);
+          completedDocumentIds.push(chunk.documentId);
+        }
+      }
+      if (completedDocumentIds.length > 0) {
+        await updateDocumentsStatus(completedDocumentIds, "indexed");
+      }
+      await patchRun2({
+        processedChunks: processed,
+        indexedDocumentCount: indexedDocumentIds.size,
+        status: "embedding",
+        warning: void 0
+      });
     }
     await patchRun2({
       status: "completed",
@@ -4499,16 +4544,17 @@ async function runIndexer(runId, config, previousRun = null) {
       finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
       durationMs: Date.now() - started
     });
-    await updateDocumentsStatus(
-      docs.map((d) => d.id),
-      "indexed"
-    );
   } catch (err) {
-    const message = err instanceof Error ? err.message : "Indexing failed unexpectedly.";
+    const rawMessage = err instanceof Error ? err.message : "Indexing failed unexpectedly.";
+    const completed = await readDocuments().then(
+      (documents) => documents.filter((document) => document.status === "indexed").length
+    );
+    const message = isRateLimitError(err) ? `Embedding quota is still unavailable. ${completed} document${completed === 1 ? "" : "s"} remain indexed; retry the unindexed files when your provider quota resets. ${rawMessage}` : rawMessage;
     await patchRun(
       {
         status: "failed",
         error: message,
+        indexedDocumentCount: completed,
         finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
         durationMs: Date.now() - started
       },

package/dist/{tool-loader-AM6O3TCX.js → tool-loader-K33ISDZP.js}

@@ -3,8 +3,8 @@ import {
   isToolLoaded,
   loadTool,
   unloadTool
-} from "./chunk-EVFK76V4.js";
-import "./chunk-UCYL66UI.js";
+} from "./chunk-STUKQORW.js";
+import "./chunk-7HVIRRTX.js";
 import "./chunk-3RG5ZIWI.js";
 export {
   hasCapability,

package/dist/{tool-registry-ED45DB5F.js → tool-registry-XQVQ6KQF.js}

@@ -6,7 +6,7 @@ import {
   getToolById,
   getToolsWithCapability,
   invalidateRegistryCache
-} from "./chunk-UCYL66UI.js";
+} from "./chunk-7HVIRRTX.js";
 import "./chunk-3RG5ZIWI.js";
 export {
   buildRegistry,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larkup/cli",
-  "version": "0.2.16",
+  "version": "0.2.17",
   "publishConfig": {
     "access": "public"
   },
@@ -29,9 +29,9 @@
     "cli-table3": "^0.6.5",
     "commander": "^15.0.0",
     "zod": "^4.4.3",
-    "@larkup/core": "0.3.1",
-    "@larkup/marketplace": "0.1.16",
-    "@larkup/vector-stores": "0.1.24"
+    "@larkup/core": "0.3.2",
+    "@larkup/vector-stores": "0.1.24",
+    "@larkup/marketplace": "0.1.17"
   },
   "devDependencies": {
     "tsup": "^8.0.0",