@plune-ai/cli 0.24.0 → 0.26.0

package/dist/reporter-core.js

@@ -33,8 +33,8 @@ function loadToken() {
 // src/reporter-core/client.ts
 import { createReadStream, openAsBlob } from "fs";
 import { pipeline } from "stream";
-import { request as httpRequest } from "http";
-import { request as httpsRequest } from "https";
+import { Agent as HttpAgent, request as httpRequest } from "http";
+import { Agent as HttpsAgent, request as httpsRequest } from "https";
 var BACKOFF_BASE_MS = 300;
 var BACKOFF_CAP_MS = 1e4;
 var DEFAULT_RETRIES = 2;
@@ -46,12 +46,14 @@ function parseRetryAfter(raw, now = Date.now()) {
   if (!Number.isFinite(seconds)) return null;
   return Math.min(Math.max(seconds, 0) * 1e3, RETRY_AFTER_CAP_MS);
 }
+var POOLS = { "http:": new HttpAgent({ keepAlive: true }), "https:": new HttpsAgent({ keepAlive: true }) };
 var IDLE_MS = 3e5;
 function nodeTransport(url, init, idleMs = IDLE_MS) {
   return new Promise((resolve2, reject) => {
     const target = new URL(url);
     const send = target.protocol === "https:" ? httpsRequest : httpRequest;
-    const req = send(target, { method: init.method, headers: init.headers }, (res) => {
+    const agent = POOLS[target.protocol === "https:" ? "https:" : "http:"];
+    const req = send(target, { method: init.method, headers: init.headers, agent }, (res) => {
       const chunks = [];
       res.on("data", (chunk2) => chunks.push(chunk2));
       res.on("error", reject);
@@ -112,14 +114,25 @@ var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 async function detailOf(res) {
   try {
     const body = await res.json();
+    const q = body.quota;
     return {
+      ...typeof body.shardsExpected === "number" && Number.isInteger(body.shardsExpected) ? { shardsExpected: body.shardsExpected } : {},
       detail: typeof body.error === "string" ? body.error : "",
+      ...typeof q === "object" && q !== null && !Array.isArray(q) ? { quota: q } : {},
       ...typeof body.code === "string" && /^[a-z0-9_]{1,60}$/.test(body.code) ? { code: body.code } : {}
     };
   } catch {
     return { detail: "" };
   }
 }
+function describeQuota(q) {
+  const num = (v) => typeof v === "number" && Number.isFinite(v) ? v : void 0;
+  const kind = typeof q["kind"] === "string" ? q["kind"].slice(0, 60) : "limit";
+  const [used, limit] = [num(q["used"]), num(q["limit"])];
+  const parts = [used !== void 0 && limit !== void 0 ? `${kind}: ${used} of ${limit} used` : kind];
+  if (typeof q["resetsAt"] === "string" && q["resetsAt"] !== "") parts.push(`resets ${q["resetsAt"].slice(0, 40)}`);
+  return parts.join(", ");
+}
 function classify(status) {
   if (status === 401 || status === 403) return "auth";
   if (status === 409) return "conflict";
@@ -184,12 +197,12 @@ function createClient(opts) {
       }
       const said = await detailOf(res);
       const kind = classify(res.status);
-      last = { kind, status: res.status, detail: scrub(said.detail), ...said.code !== void 0 ? { code: said.code } : {} };
+      last = { kind, status: res.status, detail: scrub(said.detail), ...said.code !== void 0 ? { code: said.code } : {}, ...said.shardsExpected !== void 0 ? { shardsExpected: said.shardsExpected } : {} };
       if (kind !== "unavailable") break;
       const said429 = parseRetryAfter(res.headers.get("retry-after"), now());
-      if (res.status === 429 && said429 === null) {
+      if (res.status === 429 && (said.quota !== void 0 || said429 === null)) {
         last.kind = "invalid";
-        last.detail = "quota exceeded (no Retry-After)";
+        last.detail = `quota exceeded (${said.quota === void 0 ? "no Retry-After" : scrub(describeQuota(said.quota))})`;
         if (said.detail) last.detail += ` \u2014 ${scrub(said.detail)}`;
         break;
       }
@@ -202,6 +215,7 @@ function createClient(opts) {
     post: (path3, body, retries) => send("POST", path3, { body: JSON.stringify(body), type: "application/json" }, retries),
     upload: (path3, bytes, contentType) => send("POST", path3, { body: bytes, type: contentType }, uploadRetries ?? DEFAULT_RETRIES),
     del: (path3) => send("DELETE", path3),
+    get: (path3) => send("GET", path3),
     async put(url, headers, source) {
       const putRetries = uploadRetries ?? 3;
       let last = { ok: false, status: null, code: "network" };
@@ -240,7 +254,18 @@ function createClient(opts) {
 // src/reporter-core/env.ts
 import * as path from "path";
 var UNSUPPORTED_VARS = ["PLUNE_GROUP"];
-var UPLOAD_CONCURRENCY = { min: 1, max: 32, default: 4 };
+var SHARDS_MAX = 1e3;
+function shardOf(env) {
+  const raw = value(env, "PLUNE_SHARD");
+  if (raw === void 0) return {};
+  const m = /^(\d{1,4})\/(\d{1,4})$/.exec(raw);
+  const [k, n] = m ? [Number(m[1]), Number(m[2])] : [0, 0];
+  if (k >= 1 && k <= n && n <= SHARDS_MAX) return { shard: { index: k, total: n } };
+  return {
+    error: `PLUNE_SHARD=${JSON.stringify(raw.slice(0, 40))} is not "k/N" with 1 <= k <= N <= ${SHARDS_MAX} (for example 2/4) - nothing was sent.`
+  };
+}
+var UPLOAD_CONCURRENCY = { min: 1, max: 32, default: 16 };
 var UPLOAD_RETRIES = { min: 0, max: 20, default: 6 };
 function value(env, name) {
   const raw = env[name]?.trim();
@@ -284,6 +309,7 @@ function readEnv(env = process.env) {
   const labels = list(env, "PLUNE_LABELS");
   const offerDiscovered = flag(env, "PLUNE_CREATE");
   const full = flag(env, "PLUNE_FULL_RUN");
+  const { shard, error: shardError } = shardOf(env);
   return {
     config: {
       ...externalKey !== void 0 ? { externalKey } : {},
@@ -304,7 +330,11 @@ function readEnv(env = process.env) {
     // Two names for two situations that end the same way. `SHARED_RUN` says other processes are
     // reporting into this run; `PROCEED` says the job will close it on its own schedule. Either
     // way this process must not be the one to call it finished.
-    keepOpen: flag(env, "PLUNE_SHARED_RUN") || flag(env, "PLUNE_PROCEED"),
+    // `PLUNE_SHARD` implies it: the run is joined by the shared key and this process does not close it
+    // by itself; `leaveOpen` reports the shard instead.
+    keepOpen: flag(env, "PLUNE_SHARED_RUN") || flag(env, "PLUNE_PROCEED") || shard !== void 0 || shardError !== void 0,
+    ...shard !== void 0 ? { shard } : {},
+    ...shardError !== void 0 ? { shardError } : {},
     ignored: UNSUPPORTED_VARS.filter((name) => value(env, name) !== void 0),
     warnings
   };
@@ -514,6 +544,7 @@ var RESOLVE_CHUNK = 500;
 var BATCH_MAX = 500;
 var DISCOVER_MAX = 500;
 var BATCH_DEFAULT = 500;
+var SEND_CONCURRENCY = 4;
 var EXPECTED_MAX = 1e4;
 var KEY_MAX = 1024;
 var TITLE_MAX = 300;
@@ -651,6 +682,7 @@ function warnIfDropped(sent, run, log) {
 }
 async function startRun(passed, declared) {
   const env = readEnv();
+  if (env.shardError !== void 0) throw new Error(env.shardError);
   const stated = Object.fromEntries(Object.entries(passed).filter(([, v]) => v !== void 0));
   const cfg = { ...env.config, ...stated };
   const log = cfg.log ?? ((line) => void process.stderr.write(`${line}
@@ -692,6 +724,9 @@ async function startRun(passed, declared) {
   const resolved = /* @__PURE__ */ new Map();
   let buffer = [];
   const discoveries = /* @__PURE__ */ new Map();
+  const held = /* @__PURE__ */ new Map();
+  let createdWithResult = 0;
+  let createdUndelivered = 0;
   let runId = null;
   let joined = false;
   let offline = token === "";
@@ -820,6 +855,7 @@ async function startRun(passed, declared) {
       specRef: result.specRef,
       rawStatus: result.rawStatus
     });
+    held.set(result.keys[0].value, result);
   }
   async function offer() {
     if (discoveries.size === 0 || offline || runId === null) return;
@@ -838,7 +874,24 @@ async function startRun(passed, declared) {
         return;
       }
       stats.offered += out.body.results.filter((r) => r?.outcome === "queued").length;
-      stats.created += out.body.results.filter((r) => r?.outcome === "created").length;
+      const made = out.body.results.filter((r) => r?.outcome === "created");
+      stats.created += made.length;
+      const submissions = [];
+      const filesByKey = /* @__PURE__ */ new Map();
+      for (const r of made) {
+        const pending = typeof r.id === "string" && r.id !== "" ? held.get(r.key?.value) : void 0;
+        if (pending === void 0) continue;
+        held.delete(r.key.value);
+        const { keys: _keys, files, ...rest } = pending;
+        submissions.push({ ...rest, testCaseId: r.id });
+        if (files !== void 0) filesByKey.set(pending.resultKey, files);
+      }
+      if (submissions.length > 0) {
+        const before = stats.deferred;
+        await send(submissions, filesByKey);
+        createdWithResult += submissions.length;
+        createdUndelivered += stats.deferred - before;
+      }
     }
   }
   function skip(kind, name, why) {
@@ -1245,7 +1298,29 @@ async function startRun(passed, declared) {
     else await uploadBatched(jobs);
     uploadMs += Date.now() - started;
   }
+  const sending = /* @__PURE__ */ new Set();
+  let sendError;
+  let answered = false;
+  async function dispatch(work) {
+    while (sending.size >= (answered ? SEND_CONCURRENCY : 1)) await Promise.race(sending);
+    const task = work().catch((err) => {
+      sendError ??= err;
+    }).finally(() => sending.delete(task));
+    sending.add(task);
+  }
+  async function settle() {
+    while (sending.size > 0) await Promise.all([...sending]);
+    if (sendError !== void 0) {
+      const err = sendError;
+      sendError = void 0;
+      throw err;
+    }
+  }
   async function flush() {
+    await flushBuffer();
+    await settle();
+  }
+  async function flushBuffer() {
     if (buffer.length === 0) return;
     const pending = buffer;
     buffer = [];
@@ -1253,6 +1328,9 @@ async function startRun(passed, declared) {
       defer(pending);
       return;
     }
+    await dispatch(() => deliver(pending));
+  }
+  async function deliver(pending) {
     await resolveKeys(pending.flatMap((p) => p.keys));
     const submissions = [];
     const unanswered = [];
@@ -1275,6 +1353,9 @@ async function startRun(passed, declared) {
       progress();
       return;
     }
+    await send(submissions, filesByKey);
+  }
+  async function send(submissions, filesByKey) {
     for (const part of packed(submissions)) {
       if (offline) {
         defer(part);
@@ -1290,6 +1371,7 @@ async function startRun(passed, declared) {
         for (const k of told) stats[k] += counts[k];
         progress();
         await upload(out.ok ? out.body?.items : void 0, filesByKey);
+        answered = true;
         continue;
       }
       if (out.ok) {
@@ -1318,7 +1400,10 @@ async function startRun(passed, declared) {
     if (stats.rejected > 0) parts.push(`${stats.rejected} rejected`);
     if (stats.unresolved > 0) parts.push(`${stats.unresolved} with no matching test case`);
     if (stats.offered > 0) parts.push(`${stats.offered} offered for review`);
-    if (stats.created > 0) parts.push(`${stats.created} added as cases (trusted source)`);
+    if (stats.created > 0) {
+      const recorded = createdWithResult === stats.created && createdUndelivered === 0;
+      parts.push(`${stats.created} added as cases (trusted source)${recorded ? " and their results recorded" : ""}`);
+    }
     const kinds = [stats.screenshots, stats.textFiles, stats.otherFiles];
     const sum = (what) => kinds.reduce((n, counts) => n + counts[what], 0);
     if (bucketSeen) {
@@ -1348,6 +1433,7 @@ async function startRun(passed, declared) {
       log(`::warning::${stats.deferred} result(s) were not delivered to Plune \u2014 see the reporting step's log.`);
     }
   }
+  const shardLines = [];
   return {
     get runId() {
       return runId;
@@ -1360,7 +1446,7 @@ async function startRun(passed, declared) {
     },
     async add(result) {
       buffer.push({ ...result, keys: bound(result.keys) });
-      if (buffer.length >= batchSize) await flush();
+      if (buffer.length >= batchSize) await flushBuffer();
     },
     flush,
     async finish(reason) {
@@ -1385,15 +1471,43 @@ async function startRun(passed, declared) {
       summarise();
       return frozen;
     },
-    async leaveOpen() {
-      if (done) return;
+    shardLines,
+    async leaveOpen(opts) {
+      if (done) return void 0;
       done = true;
       await flush();
       await offer();
-      if (runId !== null) {
+      let frozen;
+      const after = [];
+      const shard = env.shard;
+      let counted = false;
+      let mismatch;
+      if (runId !== null && shard !== void 0 && !offline) {
+        const out = await client.post(`/v1/runs/${runId}/events`, { event: "finish", shardIndex: shard.index, shardsExpected: shard.total });
+        if (out.ok && typeof out.body?.shardsReported === "number") {
+          counted = true;
+          frozen = out.body.summary ?? void 0;
+          const n = out.body.shardsExpected ?? shard.total;
+          if (out.body.run?.statusEvent === "finished") after.push("plune: run closed");
+          after.push(`plune: shard ${shard.index} of ${shard.total} done \u2014 ${out.body.shardsReported} of ${n} shards reported`);
+        } else if (out.ok || out.kind === "invalid") {
+          log('plune: the platform does not support shard auto-close yet \u2014 the run is left open for "plune run finish".');
+        } else if (out.code === "shards_mismatch") {
+          mismatch = new Error(
+            `PLUNE_SHARD=${shard.index}/${shard.total} says ${shard.total} shards, but run ${runId} already expects ${out.shardsExpected ?? "a different number of"} \u2014 every shard of one run must use the same N. The results are uploaded; the run is left open.`
+          );
+        } else {
+          log(`plune: could not report shard ${shard.index} of ${shard.total} to run ${runId} (${out.detail || out.kind}) \u2014 the results are uploaded; close the run with "plune run finish ${runId}".`);
+        }
+      }
+      if (runId !== null && !counted) {
         log(`plune: run ${runId} is still open \u2014 close it with "plune run finish ${runId}".`);
       }
       summarise();
+      shardLines.push(...after);
+      if (opts?.holdShardLines !== true) for (const line of after) log(line);
+      if (mismatch !== void 0) throw mismatch;
+      return frozen;
     }
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.24.0",
+  "version": "0.26.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",