@arbidocs/cli 0.3.49 → 0.3.51

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v0.3.51
+
+[compare changes](https://github.com/arbicity/ARBI-frontend/compare/v0.3.50...HEAD)
+
+### 🔥 Performance
+
+- Stop auto-refetching workspace documents/conversations/tags ([#650](https://github.com/arbicity/ARBI-frontend/pull/650), [#651](https://github.com/arbicity/ARBI-frontend/pull/651))
+
+### 🩹 Fixes
+
+- Exclude /arbi-files/ from static-asset nginx location so image uploads reach MinIO ([#646](https://github.com/arbicity/ARBI-frontend/pull/646))
+- Make workspace switch sequential and race-free ([#648](https://github.com/arbicity/ARBI-frontend/pull/648), [#649](https://github.com/arbicity/ARBI-frontend/pull/649))
+- Omit doc_ext_ids by default in retrieve, cap explicit selection at 5000 ([#652](https://github.com/arbicity/ARBI-frontend/pull/652))
+
+## v0.3.50
+
+[compare changes](https://github.com/arbicity/ARBI-frontend/compare/v0.3.49...HEAD)
+
+### 🚀 Enhancements
+
+- Use FileX icon for empty document status ([#643](https://github.com/arbicity/ARBI-frontend/pull/643))
+- **cli:** Improve reprocess progress with two-phase reporting ([#584](https://github.com/arbicity/ARBI-frontend/pull/584))
+
 ## v0.3.49
 
 [compare changes](https://github.com/arbicity/ARBI-frontend/compare/v0.3.48...HEAD)

package/dist/index.js

@@ -3641,7 +3641,7 @@ function getLatestVersion(skipCache = false) {
   }
 }
 function getCurrentVersion() {
-  return "0.3.49";
+  return "0.3.51";
 }
 function readChangelog(fromVersion, toVersion) {
   try {
@@ -3694,17 +3694,17 @@ function showChangelog(fromVersion, toVersion) {
 async function checkForUpdates(autoUpdate) {
   try {
     const latest = getLatestVersion();
-    if (!latest || latest === "0.3.49") return;
+    if (!latest || latest === "0.3.51") return;
     if (autoUpdate) {
       warn(`
-Your arbi version is out of date (${"0.3.49"} \u2192 ${latest}). Updating...`);
+Your arbi version is out of date (${"0.3.51"} \u2192 ${latest}). Updating...`);
       child_process.execSync("npm install -g @arbidocs/cli@latest", { stdio: "inherit" });
-      showChangelog("0.3.49", latest);
+      showChangelog("0.3.51", latest);
       console.log(`Updated to ${latest}.`);
     } else {
       warn(
         `
-Your arbi version is out of date (${"0.3.49"} \u2192 ${latest}).
+Your arbi version is out of date (${"0.3.51"} \u2192 ${latest}).
 Run "arbi update" to upgrade, or "arbi update auto" to always stay up to date.`
       );
     }
@@ -3714,9 +3714,9 @@ Run "arbi update" to upgrade, or "arbi update auto" to always stay up to date.`
 function hintUpdateOnError() {
   try {
     const cached = readCache();
-    if (cached && cached.latest !== "0.3.49") {
+    if (cached && cached.latest !== "0.3.51") {
       warn(
-        `Your arbi version is out of date (${"0.3.49"} \u2192 ${cached.latest}). Run "arbi update".`
+        `Your arbi version is out of date (${"0.3.51"} \u2192 ${cached.latest}). Run "arbi update".`
       );
     }
   } catch {
@@ -5338,10 +5338,16 @@ function registerDocsCommand(program2) {
     "--status-interval <seconds>",
     "Seconds between progress lines (0 to disable periodic printing).",
     "5"
-  ).option("-v, --verbose", "Print a log line after each batch in addition to periodic stats.").option("-q, --quiet", "Suppress periodic progress lines \u2014 only print the final summary.").action(
+  ).option("-v, --verbose", "Print a log line after each batch in addition to periodic stats.").option("-q, --quiet", "Suppress periodic progress lines \u2014 only print the final summary.").option("-W, --watch", "Watch document processing progress after submission").option("--no-watch", "Skip watching document processing").action(
     (ids, opts) => runAction(async () => {
-      const { arbi } = await resolveWorkspace(void 0, { skipNotifications: true });
+      const isInteractive = process.stdout.isTTY === true;
+      const watchPref = getConfig()?.watch !== false;
+      const willWatch = opts.watch === false ? false : opts.watch === true || watchPref && isInteractive;
+      const { arbi, config, accessToken } = await resolveWorkspace(void 0, {
+        skipNotifications: true
+      });
       let docIds;
+      const docNames = /* @__PURE__ */ new Map();
       if (ids && ids.length > 0) {
         if (opts.ext || opts.folder) {
           warn("--ext/--folder are ignored when explicit IDs are given.");
@@ -5366,6 +5372,9 @@ function registerDocsCommand(program2) {
           );
         }
         docIds = filtered.map((d) => d.external_id);
+        for (const d of filtered) {
+          docNames.set(d.external_id, d.file_name ?? "Unnamed");
+        }
       } else {
         error("Provide document IDs or use --status to select documents to reprocess.");
         process.exit(1);
@@ -5406,11 +5415,21 @@ function registerDocsCommand(program2) {
         const pad = (n) => n.toString().padStart(2, "0");
         return `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
       };
+      let procCompleted = 0;
+      let procFailed = 0;
+      const procPending = /* @__PURE__ */ new Set();
+      const procFailedDetails = /* @__PURE__ */ new Map();
       const printStatus = (prefix = "") => {
         const now = Date.now();
         const elapsed = now - startMs;
-        const rate = submitted > 0 ? submitted / elapsed * 1e3 : 0;
-        const bar = `[${formatClock(now)}] ${submitted}/${total}  patched=${succeeded} failed=${failed}  rate=${rate.toFixed(1)}/s  elapsed=${formatDuration(elapsed)}`;
+        const submitRate = submitted > 0 ? submitted / elapsed * 1e3 : 0;
+        let bar = `[${formatClock(now)}] submitted=${submitted}/${total}  api_ok=${succeeded} api_err=${failed}  rate=${submitRate.toFixed(1)}/s  elapsed=${formatDuration(elapsed)}`;
+        if (willWatch) {
+          const procRate = procCompleted > 0 ? procCompleted / elapsed * 1e3 : 0;
+          const pendingCount = procPending.size;
+          const eta = procRate > 0 ? pendingCount / procRate : 0;
+          bar += `  |  proc_ok=${procCompleted} proc_err=${procFailed} proc_pending=${pendingCount}  rate(proc)=${procRate.toFixed(1)}/s  eta(processing)=${formatDuration(eta * 1e3)}`;
+        }
         console.log(`${prefix}${bar}`);
       };
       let statusTimer;
@@ -5437,6 +5456,65 @@ function registerDocsCommand(program2) {
           return false;
         }
       };
+      let conn = null;
+      let onWatchDone = null;
+      let watchDonePromise = null;
+      let submissionDone = false;
+      if (willWatch) {
+        watchDonePromise = new Promise((r) => {
+          onWatchDone = r;
+        });
+        try {
+          conn = await sdk.connectWithReconnect({
+            baseUrl: config.baseUrl,
+            accessToken,
+            onReconnecting: (attempt, max) => {
+              warn(`  WebSocket reconnecting (${attempt}/${max})...`);
+            },
+            onReconnectFailed: () => {
+              error('  WebSocket reconnection failed. Run "arbi watch" to resume monitoring.');
+            },
+            onMessage: (msg) => {
+              if (!client.isMessageType(msg, "task_update")) return;
+              if (!procPending.has(msg.doc_ext_id)) return;
+              const docName = docNames.get(msg.doc_ext_id) || msg.file_name;
+              const extra = msg;
+              if (msg.status === "failed") {
+                procFailed += 1;
+                procPending.delete(msg.doc_ext_id);
+                const wsReason = extra.error_reason || extra.status_details || extra.detail || "";
+                const logFailure = (reason) => {
+                  const suffix = reason ? ` \u2014 ${reason}` : "";
+                  procFailedDetails.set(msg.doc_ext_id, `${docName}${suffix}`);
+                  console.log(`  ${docName}: ${chalk2__default.default.red("processing failed")}${suffix}`);
+                  if (submissionDone && procPending.size === 0) conn?.close();
+                };
+                if (wsReason) {
+                  logFailure(wsReason);
+                } else {
+                  sdk.documents.getDocuments(arbi, [msg.doc_ext_id]).then((docs) => {
+                    const d = docs[0];
+                    logFailure(d?.error_message || "");
+                  }).catch(() => logFailure(""));
+                }
+              } else if (msg.status === "completed" || msg.status === "skipped" || msg.status === "empty") {
+                procCompleted += 1;
+                procPending.delete(msg.doc_ext_id);
+              }
+              if (msg.status !== "failed" && submissionDone && procPending.size === 0) {
+                conn?.close();
+              }
+            },
+            onClose: () => {
+              onWatchDone?.();
+            }
+          });
+        } catch {
+          warn("WebSocket connection failed. Proceeding without processing watch.");
+          conn = null;
+          watchDonePromise = null;
+        }
+      }
       try {
         for (let i = 0; i < docIds.length; i += batchSize) {
           const batch = docIds.slice(i, i + batchSize);
@@ -5451,6 +5529,7 @@ function registerDocsCommand(program2) {
           if (batchOk) {
             succeeded += batch.length;
             submitted += batch.length;
+            for (const id of batch) procPending.add(id);
             if (opts.verbose) {
               console.log(`  [${submitted}/${total}] Triggered reprocessing...`);
             }
@@ -5465,6 +5544,7 @@ function registerDocsCommand(program2) {
               const docOk = await submitBatch([id]);
               if (docOk) {
                 succeeded += 1;
+                procPending.add(id);
               } else {
                 failed += 1;
                 failedIds.push(id);
@@ -5482,10 +5562,11 @@ function registerDocsCommand(program2) {
       } finally {
         if (statusTimer) clearInterval(statusTimer);
       }
+      submissionDone = true;
       console.log("");
       printStatus();
       if (failed > 0) {
-        warn(`${failed} document(s) failed to reprocess.`);
+        warn(`${failed} document(s) failed to submit.`);
         const preview = failedIds.slice(0, 20);
         preview.forEach((id) => console.log(`  ${chalk2__default.default.red("\u2717")} ${id}`));
         if (failedIds.length > preview.length) {
@@ -5493,7 +5574,60 @@ function registerDocsCommand(program2) {
         }
       }
       if (succeeded > 0) {
-        success(`Triggered reprocessing for ${succeeded} document(s).`);
+        success(`Submitted ${succeeded} document(s) for reprocessing.`);
+      }
+      if (watchDonePromise && conn && procPending.size > 0) {
+        console.log(chalk2__default.default.bold(`
+Watching processing for ${procPending.size} document(s)...`));
+        let watchTimer;
+        if (!opts.quiet && statusIntervalMs > 0) {
+          watchTimer = setInterval(() => printStatus("  "), statusIntervalMs);
+        }
+        await watchDonePromise;
+        if (watchTimer) clearInterval(watchTimer);
+        const totalElapsed = formatDuration(Date.now() - startMs);
+        if (procFailedDetails.size > 0) {
+          console.log(
+            `
+Processing complete: completed=${procCompleted} failed=${procFailed}  elapsed=${totalElapsed}`
+          );
+          error("Failed documents:");
+          for (const [, detail] of procFailedDetails) {
+            error(`  ${detail}`);
+          }
+        } else if (procPending.size > 0) {
+          warn(`
+Disconnected. ${procPending.size} document(s) still processing.`);
+          dim('Run "arbi watch" to continue monitoring, or "arbi docs" to check status.');
+        } else {
+          success(
+            `
+Processing complete: completed=${procCompleted} failed=0  elapsed=${totalElapsed}  ${chalk2__default.default.green("\u2713 all ok")}`
+          );
+        }
+      } else if (succeeded > 0 && !willWatch) {
+        dim(
+          'Tip: Use -W/--watch to monitor processing progress, or run "arbi docs" to check status.'
+        );
+      } else if (watchDonePromise && conn && procPending.size === 0) {
+        conn.close();
+        await watchDonePromise;
+        const totalElapsed = formatDuration(Date.now() - startMs);
+        if (procFailedDetails.size > 0) {
+          console.log(
+            `
+Processing complete: completed=${procCompleted} failed=${procFailed}  elapsed=${totalElapsed}`
+          );
+          error("Failed documents:");
+          for (const [, detail] of procFailedDetails) {
+            error(`  ${detail}`);
+          }
+        } else {
+          success(
+            `
+Processing complete: completed=${procCompleted} failed=0  elapsed=${totalElapsed}  ${chalk2__default.default.green("\u2713 all ok")}`
+          );
+        }
       }
     })()
   );
@@ -6790,6 +6924,7 @@ function printCitationDetail(num, resolved) {
     console.log(chalk2__default.default.dim("  (no passage data available)"));
   }
 }
+var MAX_INDIVIDUALLY_SELECTED_DOCS = 5e3;
 function chunkScore(chunk) {
   return chunk.metadata.rerank_score ?? chunk.metadata.score ?? 0;
 }
@@ -6802,20 +6937,28 @@ function fmtScore(score) {
   return `[${pct}%]`;
 }
 function registerFindCommand(program2) {
-  program2.command("find <query...>").description("Search documents without LLM generation (retrieve only)").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").option("-m, --mode <mode>", "Search mode: semantic, keyword, hybrid", "semantic").option("-d, --docs <ids>", "Comma-separated document IDs to search within").option("--min-score <n>", "Minimum relevance score 0-1", "0.2").option("-l, --limit <n>", "Max results to display", "20").option("--flat", "Flat list sorted by relevance (instead of grouped by document)").option("--toc <ids>", "Comma-separated doc IDs to fetch table of contents for").option("--full-context <ids>", "Comma-separated doc IDs to fetch full content for").option("--json", "Raw JSON output for scripting").action(
+  program2.command("find <query...>").description("Search documents without LLM generation (retrieve only)").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").option("-m, --mode <mode>", "Search mode: semantic, keyword, hybrid", "semantic").option(
+    "-d, --docs <ids>",
+    `Comma-separated document IDs to search within (omit to search the whole workspace; max ${MAX_INDIVIDUALLY_SELECTED_DOCS} when set)`
+  ).option("--min-score <n>", "Minimum relevance score 0-1", "0.2").option("-l, --limit <n>", "Max results to display", "20").option("--flat", "Flat list sorted by relevance (instead of grouped by document)").option("--toc <ids>", "Comma-separated doc IDs to fetch table of contents for").option("--full-context <ids>", "Comma-separated doc IDs to fetch full content for").option("--json", "Raw JSON output for scripting").action(
     (words, opts) => runAction(async () => {
       const query = words.join(" ");
       const { arbi, workspaceId } = await resolveWorkspace(opts.workspace);
       let docIds;
       if (opts.docs) {
-        docIds = opts.docs.split(",").map((id) => id.trim());
+        docIds = opts.docs.split(",").map((id) => id.trim()).filter((id) => id.length > 0);
+        if (docIds.length > MAX_INDIVIDUALLY_SELECTED_DOCS) {
+          console.error(
+            chalk2__default.default.red(
+              `Error: ${docIds.length.toLocaleString()} document IDs supplied \u2014 maximum is ${MAX_INDIVIDUALLY_SELECTED_DOCS.toLocaleString()}.
+Narrow your selection, or omit -d to search the whole workspace.`
+            )
+          );
+          process.exitCode = 1;
+          return;
+        }
       } else {
-        const docs = await sdk.documents.listDocuments(arbi);
-        docIds = docs.map((d) => d.external_id);
-      }
-      if (docIds.length === 0) {
-        console.log("No documents found in workspace.");
-        return;
+        docIds = [];
       }
       const searchMode = opts.mode ?? "semantic";
       const tocDocIds = opts.toc ? opts.toc.split(",").map((id) => id.trim()) : void 0;
@@ -8606,7 +8749,7 @@ console.info = (...args) => {
   _origInfo(...args);
 };
 var program = new commander.Command();
-program.name("arbi").description("ARBI CLI \u2014 interact with ARBI from the terminal").version("0.3.49");
+program.name("arbi").description("ARBI CLI \u2014 interact with ARBI from the terminal").version("0.3.51");
 registerConfigCommand(program);
 registerLoginCommand(program);
 registerRegisterCommand(program);