exomind 0.6.0 → 0.7.1

package/dist/cli.js

@@ -3119,7 +3119,7 @@ var {
 // package.json
 var package_default = {
   name: "exomind",
-  version: "0.6.0",
+  version: "0.7.1",
   description: "ExoMind \u8DE8\u5E73\u53F0\u547D\u4EE4\u884C\u5BA2\u6237\u7AEF \u2014 \u901A\u8FC7 REST \u4E0E ExoMind \u77E5\u8BC6\u5E93\u4EA4\u4E92(ingest/query/search/review),\u66FF\u4EE3 Windows \u4E0D\u53EF\u7528\u7684 MCP \u5BA2\u6237\u7AEF\u3002",
   bin: {
     exomind: "dist/cli.js"
@@ -3917,31 +3917,69 @@ async function mapWithConcurrency(items, concurrency, worker) {
 }
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 var suspended = false;
-async function ingestWithRetry(client, payload, timeoutMs) {
+function friendlyMsg(e) {
+  if (e instanceof ApiError) {
+    if (e.status === 504) return "\u670D\u52A1\u5668\u5904\u7406\u8D85\u65F6";
+    if (e.status === 502) return "\u7F51\u5173\u5F02\u5E38";
+    if (e.status === 503) return "\u670D\u52A1\u6682\u4E0D\u53EF\u7528";
+    if (e.status === 0) return "\u7F51\u7EDC\u9519\u8BEF";
+  }
+  return (e.message || String(e)).slice(0, 200);
+}
+async function ingestWithRetry(client, payload, timeoutMs, url = "/ingest") {
   let rateAttempts = 0;
   let quotaWaits = 0;
+  let transientAttempts = 0;
   while (true) {
     try {
-      return await client.post("/ingest", payload, { timeoutMs });
+      return await client.post(url, payload, { timeoutMs });
     } catch (e) {
-      if (!(e instanceof ApiError) || e.status !== 429) throw e;
-      const type = e.body?.type;
-      if (type === "daily_quota") {
-        if (++quotaWaits > 3) throw new ApiError(429, "\u914D\u989D\u8FDE\u7EED 3 \u4E2A\u81EA\u7136\u65E5\u672A\u6062\u590D,\u653E\u5F03");
-        await suspendUntilMidnight(Number(e.body?.reset ?? 0));
-        continue;
+      if (!(e instanceof ApiError)) throw e;
+      if (e.status === 429) {
+        const type = e.body?.type;
+        if (type === "daily_quota") {
+          if (++quotaWaits > 3) throw new ApiError(429, "\u914D\u989D\u8FDE\u7EED 3 \u4E2A\u81EA\u7136\u65E5\u672A\u6062\u590D,\u653E\u5F03");
+          await suspendUntilMidnight(Number(e.body?.reset ?? 0));
+          continue;
+        }
+        if (type === "rate_limit") {
+          if (++rateAttempts >= 5) throw new ApiError(429, "\u5E76\u53D1\u9650\u6D41,\u91CD\u8BD5 5 \u6B21\u4ECD\u5931\u8D25");
+          const retry = Number(e.headers["retry-after"] ?? e.body?.retry_after ?? 5);
+          process.stderr.write(dim(`  \u23F8 \u5E76\u53D1\u9650\u6D41,${retry}s \u540E\u91CD\u8BD5
+`));
+          await sleep(retry * 1e3);
+          continue;
+        }
+        throw e;
       }
-      if (type === "rate_limit") {
-        if (++rateAttempts >= 5) throw new ApiError(429, "\u5E76\u53D1\u9650\u6D41,\u91CD\u8BD5 5 \u6B21\u4ECD\u5931\u8D25");
-        const retry = Number(e.headers["retry-after"] ?? e.body?.retry_after ?? 5);
-        process.stderr.write(dim(`  \u23F8 \u5E76\u53D1\u9650\u6D41,${retry}s \u540E\u91CD\u8BD5
+      if ([502, 503, 504].includes(e.status) || e.status === 0) {
+        if (++transientAttempts > 3) throw e;
+        const backoff = Math.pow(2, transientAttempts - 1) * 1e3;
+        process.stderr.write(dim(`  \u23F8 ${friendlyMsg(e)},${backoff / 1e3}s \u540E\u91CD\u8BD5 (${transientAttempts}/3)
 `));
-        await sleep(retry * 1e3);
+        await sleep(backoff);
+        continue;
+      }
+      throw e;
+    }
+  }
+}
+async function retryTransient(fn, maxAttempts = 3) {
+  let lastErr;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (e) {
+      lastErr = e;
+      const transient = e instanceof ApiError && ([502, 503, 504].includes(e.status) || e.status === 0);
+      if (transient && attempt < maxAttempts) {
+        await sleep(Math.pow(2, attempt - 1) * 1e3);
         continue;
       }
       throw e;
     }
   }
+  throw lastErr;
 }
 async function suspendUntilMidnight(resetEpoch) {
   suspended = true;
@@ -3975,7 +4013,7 @@ async function runDirIngestest(client, opts, dir) {
   const total = files.length;
   const concurrency = Math.max(1, opts.concurrency ?? 3);
   process.stderr.write(
-    `\u76EE\u5F55 ${dir}: ${total} \u6587\u4EF6 \u2014 \u5F85\u6444\u5165 ${plan.toIngest.length},\u8DF3\u8FC7 ${plan.toSkip.length}(\u5E76\u53D1 ${concurrency})
+    `\u76EE\u5F55 ${dir}: ${total} \u6587\u4EF6 \u2014 \u5F85\u6444\u5165 ${plan.toIngest.length},\u8DF3\u8FC7 ${plan.toSkip.length}(\u5F02\u6B65,\u5E76\u53D1\u63D0\u4EA4 ${concurrency})
 `
   );
   const onSigInt = () => {
@@ -3989,56 +4027,101 @@ async function runDirIngestest(client, opts, dir) {
   let added = 0;
   let updated = 0;
   let failed = 0;
-  let started = 0;
-  const queue = plan.toIngest;
+  let degraded = 0;
+  const pending = [];
   try {
-    await mapWithConcurrency(queue, concurrency, async (f) => {
-      const prev = manifest[f.path];
-      const seq = ++started;
-      process.stderr.write(`\u23F3 [${seq}/${queue.length}] ${path4.basename(f.path)}\u2026
-`);
+    await mapWithConcurrency(plan.toIngest, concurrency, async (f) => {
       try {
         const res = await ingestWithRetry(
           client,
           { content: f.content, title: f.title, tags: opts.tag },
-          opTimeout(3e5)
+          3e4,
+          "/ingest/async"
         );
-        recordFile(manifest, f.path, f.content, f.title);
-        saveManifest(manifest);
-        if (prev) {
-          updated++;
-          process.stderr.write(`  ${green("\u2713")} \u66F4\u65B0 \u2014 \u5B9E\u4F53 ${res.entities ?? 0}/\u6982\u5FF5 ${res.concepts ?? 0}
-`);
-        } else {
-          added++;
-          process.stderr.write(`  ${green("\u2713")} \u65B0\u589E \u2014 \u5B9E\u4F53 ${res.entities ?? 0}/\u6982\u5FF5 ${res.concepts ?? 0}
-`);
-        }
+        pending.push({ file: f, jobId: res.job_id });
       } catch (e) {
         failed++;
-        process.stderr.write(`  ${red("\u2717")} ${e.message}
+        process.stderr.write(`  ${red("\u2717")} ${path4.basename(f.path)} \u63D0\u4EA4\u5931\u8D25 \u2014 ${friendlyMsg(e)}
 `);
       }
     });
+    const totalJobs = pending.length;
+    let done = 0;
+    const pollDeadline = Date.now() + opTimeout(6e5);
+    const inFlight = new Map(
+      pending.map((p) => [p.jobId, p.file])
+    );
+    while (inFlight.size > 0) {
+      if (Date.now() > pollDeadline) {
+        for (const [, f] of inFlight) {
+          failed++;
+          process.stderr.write(`  ${red("\u2717")} ${path4.basename(f.path)} \u8F6E\u8BE2\u8D85\u65F6
+`);
+        }
+        inFlight.clear();
+        break;
+      }
+      await Promise.all(
+        [...inFlight.entries()].map(async ([jobId, f]) => {
+          try {
+            const s = await retryTransient(() => client.get("/ingest/status", { job_id: jobId }));
+            if (s.status !== "done" && s.status !== "failed") return;
+            inFlight.delete(jobId);
+            done++;
+            if (s.status === "failed") {
+              failed++;
+              process.stderr.write(`  ${red("\u2717")} ${path4.basename(f.path)} \u5904\u7406\u5931\u8D25 \u2014 ${s.error || "\u672A\u77E5\u9519\u8BEF"}
+`);
+              return;
+            }
+            const prev = manifest[f.path];
+            recordFile(manifest, f.path, f.content, f.title);
+            saveManifest(manifest);
+            if (s.extracted === false) {
+              degraded++;
+              process.stderr.write(`  \u26A0 ${path4.basename(f.path)} \u964D\u7EA7 \u2014 \u539F\u6587\u5DF2\u5B58,\u5B9E\u4F53\u62BD\u53D6\u5F85\u8865\u8DD1
+`);
+            } else if (prev) {
+              updated++;
+              process.stderr.write(`  ${green("\u2713")} \u66F4\u65B0 \u2014 \u5B9E\u4F53 ${s.entities ?? 0}/\u6982\u5FF5 ${s.concepts ?? 0}
+`);
+            } else {
+              added++;
+              process.stderr.write(`  ${green("\u2713")} \u65B0\u589E \u2014 \u5B9E\u4F53 ${s.entities ?? 0}/\u6982\u5FF5 ${s.concepts ?? 0}
+`);
+            }
+          } catch {
+          }
+        })
+      );
+      if (inFlight.size > 0) {
+        process.stderr.write(`${dim("\u23F3")} [\u5B8C\u6210 ${done}/${totalJobs}] \u8F6E\u8BE2\u4E2D\u2026
+`);
+        await sleep(2e3);
+      }
+    }
     cleanupStale(manifest, dir, files);
     saveManifest(manifest);
   } finally {
     process.removeListener("SIGINT", onSigInt);
   }
-  const allUpToDate = added + updated === 0 && plan.toSkip.length > 0 && failed === 0;
+  const allUpToDate = added + updated + degraded === 0 && plan.toSkip.length > 0 && failed === 0;
   output(
-    { added, updated, skipped: plan.toSkip.length, failed, total, dir, allUpToDate },
+    { added, updated, degraded, skipped: plan.toSkip.length, failed, total, dir, allUpToDate },
     () => {
       console.log(
         green("\u2713 \u76EE\u5F55\u6444\u5165\u5B8C\u6210") + dim(
-          `: \u65B0\u589E ${added} / \u66F4\u65B0 ${updated} / \u8DF3\u8FC7 ${plan.toSkip.length} / \u5931\u8D25 ${failed} (\u5171 ${total})`
+          `: \u65B0\u589E ${added} / \u66F4\u65B0 ${updated} / \u964D\u7EA7 ${degraded} / \u8DF3\u8FC7 ${plan.toSkip.length} / \u5931\u8D25 ${failed} (\u5171 ${total})`
         )
       );
-      if (allUpToDate) {
+      if (degraded > 0) {
         console.log(
-          dim("\uFF08\u5168\u90E8\u5DF2\u662F\u6700\u65B0,\u65E0\u9700\u91CD\u6444;\u9664\u975E\u7528\u6237\u660E\u786E\u8981\u6C42\u5F3A\u5236\u5237\u65B0,\u5426\u5219\u4E0D\u8981\u52A0 --force\uFF09")
+          dim(`\uFF08${degraded} \u6761\u964D\u7EA7:\u539F\u6587\u5DF2\u5165\u5E93,\u5B9E\u4F53\u62BD\u53D6\u5F85\u8865\u2014\u2014\u53EF\u7A0D\u540E\u91CD\u8DD1\u6216 exomind ingest --backfill \u8865\u8DD1\uFF09`)
         );
       }
+      if (allUpToDate) {
+        console.log(dim("\uFF08\u5168\u90E8\u5DF2\u662F\u6700\u65B0,\u65E0\u9700\u91CD\u6444;\u9664\u975E\u7528\u6237\u660E\u786E\u8981\u6C42\u5F3A\u5236\u5237\u65B0,\u5426\u5219\u4E0D\u8981\u52A0 --force\uFF09"));
+      }
     }
   );
 }