@plune-ai/cli 0.6.0 → 0.8.0

package/dist/reporter-core.js

@@ -0,0 +1,423 @@
+// src/cli/api-url.ts
+var DEFAULT_API_URL = "https://beta-api.plune.ai";
+function resolveApiUrl(explicit) {
+  const raw = explicit ?? process.env["PLUNE_API_URL"] ?? DEFAULT_API_URL;
+  return raw.replace(/\/+$/, "");
+}
+
+// src/cli/credentials.ts
+import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { dirname, join } from "path";
+function credentialsFile() {
+  const base = process.env["XDG_CONFIG_HOME"]?.trim() || join(homedir(), ".config");
+  return join(base, "plune", "credentials.json");
+}
+function loadToken() {
+  try {
+    const parsed = JSON.parse(readFileSync(credentialsFile(), "utf-8"));
+    return typeof parsed.token === "string" && parsed.token.length > 0 ? parsed.token : null;
+  } catch {
+    return null;
+  }
+}
+
+// src/reporter-core/client.ts
+var RETRY_DELAYS_MS = [300, 900];
+var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+async function detailOf(res) {
+  try {
+    const body = await res.json();
+    return typeof body.error === "string" ? body.error : "";
+  } catch {
+    return "";
+  }
+}
+function retryAfterMs(res) {
+  const raw = res.headers.get("retry-after");
+  if (raw === null) return null;
+  const seconds = Number(raw.trim());
+  return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1e3 : null;
+}
+function classify(status) {
+  if (status === 401 || status === 403) return "auth";
+  if (status === 409) return "conflict";
+  if (status === 400) return "invalid";
+  if (status === 429 || status >= 500) return "unavailable";
+  return "other";
+}
+function createClient(opts) {
+  const doFetch = opts.fetchImpl ?? fetch;
+  const wait = opts.wait ?? sleep;
+  const base = opts.apiUrl.replace(/\/+$/, "");
+  const scrub = (text) => opts.token === "" ? text : text.split(opts.token).join("***");
+  async function post(path2, body) {
+    let last = {
+      kind: "unavailable",
+      status: null,
+      detail: "no attempt was made"
+    };
+    for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt += 1) {
+      let res;
+      try {
+        res = await doFetch(`${base}${path2}`, {
+          method: "POST",
+          headers: { "content-type": "application/json", authorization: `Bearer ${opts.token}` },
+          body: JSON.stringify(body)
+        });
+      } catch (err) {
+        last = { kind: "unavailable", status: null, detail: scrub(String(err)) };
+        if (attempt < RETRY_DELAYS_MS.length) await wait(RETRY_DELAYS_MS[attempt]);
+        continue;
+      }
+      if (res.ok) return { ok: true, status: res.status, body: await res.json() };
+      const detail = scrub(await detailOf(res));
+      const kind = classify(res.status);
+      last = { kind, status: res.status, detail };
+      if (kind !== "unavailable") break;
+      if (attempt < RETRY_DELAYS_MS.length) {
+        await wait(retryAfterMs(res) ?? RETRY_DELAYS_MS[attempt]);
+      }
+    }
+    return { ok: false, ...last };
+  }
+  return { post };
+}
+
+// src/reporter-core/env.ts
+var UNSUPPORTED_VARS = ["PLUNE_GROUP"];
+function value(env, name) {
+  const raw = env[name]?.trim();
+  return raw === void 0 || raw === "" ? void 0 : raw;
+}
+function flag(env, name) {
+  const raw = value(env, name)?.toLowerCase();
+  return raw === "1" || raw === "true" || raw === "yes" || raw === "on";
+}
+function count(env, name) {
+  const raw = value(env, name);
+  if (raw === void 0) return void 0;
+  const n = Number(raw);
+  return Number.isInteger(n) && n > 0 ? n : void 0;
+}
+function list(env, name) {
+  const raw = value(env, name);
+  if (raw === void 0) return void 0;
+  const items = raw.split(",").map((s) => s.trim()).filter((s) => s !== "");
+  return items.length === 0 ? void 0 : items;
+}
+function readEnv(env = process.env) {
+  const externalKey = value(env, "PLUNE_RUN");
+  const token = value(env, "PLUNE_TOKEN");
+  const apiUrl = value(env, "PLUNE_API_URL");
+  const fallbackPath = value(env, "PLUNE_FALLBACK");
+  const batchSize = count(env, "PLUNE_BATCH_SIZE");
+  const title = value(env, "PLUNE_RUN_TITLE");
+  const environment = value(env, "PLUNE_ENV");
+  const labels = list(env, "PLUNE_LABELS");
+  const offerDiscovered = flag(env, "PLUNE_CREATE");
+  return {
+    config: {
+      ...externalKey !== void 0 ? { externalKey } : {},
+      ...token !== void 0 ? { token } : {},
+      ...apiUrl !== void 0 ? { apiUrl } : {},
+      ...fallbackPath !== void 0 ? { fallbackPath } : {},
+      ...batchSize !== void 0 ? { batchSize } : {},
+      ...title !== void 0 ? { title } : {},
+      ...environment !== void 0 ? { environment } : {},
+      ...labels !== void 0 ? { labels } : {},
+      // Only when set: `false` here would outrank a config file that asked for it, and an unset
+      // variable is not an instruction.
+      ...offerDiscovered ? { offerDiscovered } : {}
+    },
+    // Two names for two situations that end the same way. `SHARED_RUN` says other processes are
+    // reporting into this run; `PROCEED` says the job will close it on its own schedule. Either
+    // way this process must not be the one to call it finished.
+    keepOpen: flag(env, "PLUNE_SHARED_RUN") || flag(env, "PLUNE_PROCEED"),
+    ignored: UNSUPPORTED_VARS.filter((name) => value(env, name) !== void 0)
+  };
+}
+
+// src/reporter-core/fallback.ts
+import * as fs from "fs";
+import * as path from "path";
+var DEFAULT_FALLBACK_PATH = path.join(".plune", "pending-results.jsonl");
+function appendBatch(file, batch) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const line = {
+    ts: (/* @__PURE__ */ new Date()).toISOString(),
+    runId: batch.runId,
+    externalKey: batch.externalKey,
+    results: batch.results
+  };
+  fs.appendFileSync(file, `${JSON.stringify(line)}
+`, "utf8");
+}
+
+// src/reporter-core/session.ts
+var RESOLVE_CHUNK = 500;
+var BATCH_MAX = 500;
+var DISCOVER_MAX = 500;
+var BATCH_DEFAULT = 100;
+var EXPECTED_MAX = 1e4;
+function chunk(items, size) {
+  const out = [];
+  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
+  return out;
+}
+function warnIfDropped(sent, run, log) {
+  const names = Object.keys(sent);
+  if (names.length === 0) return;
+  if (names.some((name) => run[name] != null)) return;
+  log(
+    `plune: this deployment stored none of ${names.join(", ")} \u2014 it is older than the fields. The run is otherwise reported in full.`
+  );
+}
+async function startRun(passed, expected) {
+  const env = readEnv();
+  const stated = Object.fromEntries(Object.entries(passed).filter(([, v]) => v !== void 0));
+  const cfg = { ...env.config, ...stated };
+  const log = cfg.log ?? ((line) => void process.stderr.write(`${line}
+`));
+  const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
+  const externalKey = cfg.externalKey ?? null;
+  const batchSize = Math.min(cfg.batchSize ?? BATCH_DEFAULT, BATCH_MAX);
+  const offerDiscovered = cfg.offerDiscovered ?? false;
+  const token = cfg.token ?? loadToken() ?? "";
+  const client = createClient({
+    apiUrl: resolveApiUrl(cfg.apiUrl),
+    token,
+    ...cfg.fetchImpl !== void 0 ? { fetchImpl: cfg.fetchImpl } : {}
+  });
+  const stats = {
+    accepted: 0,
+    duplicate: 0,
+    conflict: 0,
+    rejected: 0,
+    unresolved: 0,
+    offered: 0,
+    deferred: 0
+  };
+  const resolved = /* @__PURE__ */ new Map();
+  let buffer = [];
+  const discoveries = /* @__PURE__ */ new Map();
+  let runId = null;
+  let joined = false;
+  let offline = token === "";
+  let done = false;
+  if (offline) {
+    log('plune: no API token \u2014 run "plune login" first. Results will be written to the fallback file.');
+  }
+  if (env.ignored.length > 0) {
+    log(`plune: ignoring ${env.ignored.join(", ")} \u2014 a run has nowhere to store them yet.`);
+  }
+  function defer(results) {
+    appendBatch(fallbackPath, { runId, externalKey, results });
+    stats.deferred += results.length;
+  }
+  async function resolveKeys(values) {
+    const unknown = values.filter((k) => !resolved.has(k.value));
+    if (unknown.length === 0 || offline) return;
+    for (const part of chunk(unknown, RESOLVE_CHUNK)) {
+      const out = await client.post(
+        "/v1/test-cases/resolve",
+        { keys: part }
+      );
+      if (!out.ok) {
+        log(`plune: could not look up test cases (${out.detail || out.kind}).`);
+        return;
+      }
+      for (const row of out.body.results) resolved.set(row.key.value, row.testCaseId);
+    }
+  }
+  if (!offline) {
+    if (expected !== void 0) await resolveKeys(expected.flat());
+    const declared = expected?.slice(0, EXPECTED_MAX) ?? [];
+    const configuration = declared.length === 0 ? void 0 : {
+      expected: declared.flatMap((keys) => {
+        const hit = keys.map((k) => resolved.get(k.value)).find((v) => typeof v === "string");
+        if (typeof hit === "string") return [{ testCaseId: hit }];
+        return keys[0] === void 0 ? [] : [{ externalKey: keys[0] }];
+      })
+    };
+    if (expected !== void 0 && expected.length > EXPECTED_MAX) {
+      log(`plune: ${expected.length} tests is more than the run configuration holds \u2014 reporting the first ${EXPECTED_MAX}.`);
+    }
+    const described = {
+      ...cfg.title !== void 0 ? { title: cfg.title } : {},
+      ...cfg.environment !== void 0 ? { environment: cfg.environment } : {},
+      ...cfg.labels !== void 0 ? { labels: cfg.labels } : {}
+    };
+    const start = await client.post("/v1/runs", {
+      schemaVersion: 2,
+      kind: cfg.kind ?? "automated",
+      ...externalKey !== null ? { externalKey } : {},
+      ...cfg.meta !== void 0 ? { meta: cfg.meta } : {},
+      ...configuration !== void 0 ? { configuration } : {},
+      ...described
+    });
+    if (start.ok) {
+      runId = start.body.run.id;
+      joined = start.status === 200;
+      warnIfDropped(described, start.body.run, log);
+    } else {
+      log(`plune: could not start the run (${start.detail || start.kind}). Results will be written to ${fallbackPath}.`);
+    }
+  }
+  function matchFor(result) {
+    if (result.testCaseId !== void 0) return { kind: "found", testCaseId: result.testCaseId };
+    let unanswered = false;
+    for (const key of result.keys) {
+      const hit = resolved.get(key.value);
+      if (typeof hit === "string") return { kind: "found", testCaseId: hit };
+      if (hit === void 0) unanswered = true;
+    }
+    return unanswered ? { kind: "unanswered" } : { kind: "unmatched" };
+  }
+  function remember(result) {
+    if (!offerDiscovered) return;
+    if (result.title === void 0 || result.specRef === void 0) return;
+    if (result.keys.length === 0) return;
+    discoveries.set(result.keys[0].value, {
+      keys: result.keys,
+      title: result.title,
+      source: result.source,
+      specRef: result.specRef,
+      rawStatus: result.rawStatus
+    });
+  }
+  async function offer() {
+    if (discoveries.size === 0 || offline || runId === null) return;
+    for (const batch of chunk([...discoveries.values()], DISCOVER_MAX)) {
+      const out = await client.post("/v1/review-items/discovered", {
+        discovered: batch,
+        runId
+      });
+      if (!out.ok) {
+        log(`plune: could not offer ${batch.length} unknown tests for review (${out.detail || out.kind}).`);
+        return;
+      }
+      stats.offered += out.body.results.filter((r) => r.outcome === "queued").length;
+    }
+  }
+  async function flush() {
+    if (buffer.length === 0) return;
+    const pending = buffer;
+    buffer = [];
+    if (offline || runId === null) {
+      defer(pending);
+      return;
+    }
+    await resolveKeys(pending.flatMap((p) => p.keys));
+    const submissions = [];
+    const unanswered = [];
+    for (const result of pending) {
+      const match = matchFor(result);
+      if (match.kind === "found") {
+        const { keys: _keys, ...rest } = result;
+        submissions.push({ ...rest, testCaseId: match.testCaseId });
+      } else if (match.kind === "unmatched") {
+        stats.unresolved += 1;
+        remember(result);
+      } else {
+        unanswered.push(result);
+      }
+    }
+    if (unanswered.length > 0) defer(unanswered);
+    if (submissions.length === 0) return;
+    const out = await client.post(
+      `/v1/runs/${runId}/results`,
+      { results: submissions }
+    );
+    if (out.ok) {
+      stats.accepted += out.body.counts.accepted ?? 0;
+      stats.duplicate += out.body.counts.duplicate ?? 0;
+      stats.conflict += out.body.counts.conflict ?? 0;
+      stats.rejected += out.body.counts.rejected ?? 0;
+      return;
+    }
+    if (out.kind === "auth") {
+      log('plune: the API token was refused \u2014 run "plune login" with a fresh one.');
+      offline = true;
+    } else if (out.kind === "conflict") {
+      log(`plune: this run is closed \u2014 ${out.detail}`);
+    } else {
+      log(`plune: could not send results (${out.detail || out.kind}).`);
+    }
+    defer(submissions);
+  }
+  function summarise() {
+    const parts = [`${stats.accepted} accepted`];
+    if (stats.duplicate > 0) parts.push(`${stats.duplicate} already reported`);
+    if (stats.conflict > 0) parts.push(`${stats.conflict} conflicting`);
+    if (stats.rejected > 0) parts.push(`${stats.rejected} rejected`);
+    if (stats.unresolved > 0) parts.push(`${stats.unresolved} with no matching test case`);
+    if (stats.offered > 0) parts.push(`${stats.offered} offered for review`);
+    if (stats.deferred > 0) parts.push(`${stats.deferred} written to ${fallbackPath}`);
+    log(`plune: ${parts.join(" \xB7 ")}`);
+  }
+  return {
+    get runId() {
+      return runId;
+    },
+    get joined() {
+      return joined;
+    },
+    get stats() {
+      return stats;
+    },
+    async add(result) {
+      buffer.push(result);
+      if (buffer.length >= batchSize) await flush();
+    },
+    flush,
+    async finish(reason) {
+      if (done) return;
+      done = true;
+      await flush();
+      await offer();
+      if (runId !== null && !offline) {
+        const out = await client.post(`/v1/runs/${runId}/events`, {
+          event: "finish",
+          ...reason !== void 0 ? { reason } : {}
+        });
+        if (!out.ok) log(`plune: could not close run ${runId} (${out.detail || out.kind}).`);
+      }
+      summarise();
+    },
+    async leaveOpen() {
+      if (done) return;
+      done = true;
+      await flush();
+      await offer();
+      if (runId !== null) {
+        log(`plune: run ${runId} is still open \u2014 close it with "plune run finish ${runId}".`);
+      }
+      summarise();
+    }
+  };
+}
+
+// src/util/hash.ts
+import { createHash } from "crypto";
+function sha256(input) {
+  return createHash("sha256").update(input, "utf8").digest("hex");
+}
+
+// src/reporter-core/result-key.ts
+var RESULT_KEY_MAX = 200;
+var HASH_CHARS = 32;
+function resultKey(testId, retry) {
+  const plain = `${testId}#${retry}`;
+  return plain.length <= RESULT_KEY_MAX ? plain : `${sha256(testId).slice(0, HASH_CHARS)}#${retry}`;
+}
+export {
+  DEFAULT_FALLBACK_PATH,
+  RESULT_KEY_MAX,
+  UNSUPPORTED_VARS,
+  appendBatch,
+  createClient,
+  readEnv,
+  resultKey,
+  startRun
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",
@@ -28,6 +28,11 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "require": "./dist/index.cjs"
+    },
+    "./reporter-core": {
+      "types": "./dist/reporter-core.d.ts",
+      "import": "./dist/reporter-core.js",
+      "require": "./dist/reporter-core.cjs"
     }
   },
   "files": [
@@ -44,10 +49,14 @@
     "test": "vitest run --coverage",
     "test:watch": "vitest",
     "test:e2e": "tsup && vitest run -c vitest.e2e.config.ts",
-    "lint": "eslint \"src/**/*.ts\" --no-error-on-unmatched-pattern",
+    "lint": "eslint \"src/**/*.ts\" \"packages/*/src/**/*.ts\" --no-error-on-unmatched-pattern",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "typecheck": "tsc --noEmit",
+    "typecheck:playwright": "pnpm --filter @plune-ai/playwright typecheck",
+    "test:playwright": "pnpm --filter @plune-ai/playwright test",
+    "test:e2e:playwright": "pnpm --filter @plune-ai/playwright test:e2e",
+    "build:playwright": "pnpm --filter @plune-ai/playwright build",
     "prepublishOnly": "tsup"
   },
   "dependencies": {
@@ -59,7 +68,7 @@
     "dotenv": "^16.4.0",
     "openai": "^6.42.0",
     "yaml": "^2.9.0",
-    "zod": "^3.23.0"
+    "zod": "^4.5.4"
   },
   "devDependencies": {
     "@huggingface/transformers": "4.2.0",