@plune-ai/cli 0.30.2 → 0.31.0

package/dist/reporter-core.d.ts

@@ -324,6 +324,12 @@ interface ReporterConfig {
     environment?: string;
     /** How the run is marked — smoke, nightly, a team. */
     labels?: string[];
+    /**
+     * The combination this job runs, group name → value label (`PLUNE_CONFIGURATION`, plune-ai/plune#1251):
+     * sent as `configuration` on every results batch. Not the run's own `configuration` (what it expects).
+     * Stated, it is all there is: the variable is not read, and `{}` sends none (what a replay passes for a line that recorded none).
+     */
+    configuration?: Record<string, string>;
     /** Results per request. The platform's ceiling is 500. */
     batchSize?: number;
     /** Files in flight at once (`PLUNE_UPLOAD_CONCURRENCY`, 1 to 32, default 16). */
@@ -445,6 +451,8 @@ interface EnvSettings {
     shard?: ShardOf;
     /** `PLUNE_SHARD` was set to something that is not `k/N`. A configuration error: the caller refuses to start. */
     shardError?: string;
+    /** `PLUNE_CONFIGURATION` was set to something that is not `Group=Value;…`. Refused the same way. */
+    configurationError?: string;
     /** `PLUNE_IMPORT_TIMEOUT` in seconds, when set and usable. */
     importTimeout?: number;
     /** Variables that were set and have nowhere to land. The caller says so once. */
@@ -498,12 +506,17 @@ declare const DEFAULT_FALLBACK_PATH: string;
  * A run of 24 tests at a batch of 10 is three lines, and replay opens one run for the lines that share a
  * marker rather than one for each. A line written before the field existed has none, and is one run's
  * worth by itself — nothing in it says otherwise.
+ *
+ * `configuration` is the combination the job named (`PLUNE_CONFIGURATION`, plune-ai/plune#1251), recorded for
+ * the same reason as `meta`: replay sends the batch into the combination it ran in, whatever the machine that
+ * replays it has set. A line without it was written by a job that named none, or before the field existed.
  */
 interface DeferredBatch {
     runId: string | null;
     externalKey: string | null;
     session?: string;
     meta?: RunMeta;
+    configuration?: Record<string, string>;
     results: DeferredResult[];
 }
 /**

package/dist/reporter-core.js

@@ -265,6 +265,25 @@ function shardOf(env) {
     error: `PLUNE_SHARD=${JSON.stringify(raw.slice(0, 40))} is not "k/N" with 1 <= k <= N <= ${SHARDS_MAX} (for example 2/4) - nothing was sent.`
   };
 }
+function configurationOf(env) {
+  const raw = value(env, "PLUNE_CONFIGURATION");
+  if (raw === void 0) return {};
+  const pairs = [];
+  for (const pair of raw.split(";").map((s) => s.trim()).filter((s) => s !== "")) {
+    const at = pair.indexOf("=");
+    const name = at < 0 ? "" : pair.slice(0, at).trim();
+    const label = at < 0 ? "" : pair.slice(at + 1).trim();
+    const twice = pairs.find(([seen]) => seen.toLowerCase() === name.toLowerCase());
+    const why = at < 0 ? 'has no "="' : name === "" ? "has no group name" : label === "" ? "has no value" : twice !== void 0 ? `names the group ${JSON.stringify(twice[0])} a second time` : void 0;
+    if (why !== void 0) {
+      return {
+        error: `PLUNE_CONFIGURATION: ${JSON.stringify(pair.slice(0, 40))} ${why} - write it as "Group=Value;Group=Value" (for example "Browser=Chrome;Operating system=Windows") - nothing was sent.`
+      };
+    }
+    pairs.push([name, label]);
+  }
+  return pairs.length === 0 ? {} : { configuration: Object.fromEntries(pairs) };
+}
 var UPLOAD_CONCURRENCY = { min: 1, max: 32, default: 16 };
 var UPLOAD_RETRIES = { min: 0, max: 20, default: 6 };
 var IMPORT_TIMEOUT = { min: 0, max: 86400, default: 600 };
@@ -312,6 +331,7 @@ function readEnv(env = process.env) {
   const offerDiscovered = flag(env, "PLUNE_CREATE");
   const full = flag(env, "PLUNE_FULL_RUN");
   const { shard, error: shardError } = shardOf(env);
+  const { configuration, error: configurationError } = configurationOf(env);
   return {
     config: {
       ...externalKey !== void 0 ? { externalKey } : {},
@@ -324,6 +344,7 @@ function readEnv(env = process.env) {
       ...title !== void 0 ? { title } : {},
       ...environment !== void 0 ? { environment } : {},
       ...labels !== void 0 ? { labels } : {},
+      ...configuration !== void 0 ? { configuration } : {},
       // Only when set: `false` here would outrank a config file that asked for it, and an unset
       // variable is not an instruction.
       ...offerDiscovered ? { offerDiscovered } : {},
@@ -337,6 +358,7 @@ function readEnv(env = process.env) {
     keepOpen: flag(env, "PLUNE_SHARED_RUN") || flag(env, "PLUNE_PROCEED") || shard !== void 0 || shardError !== void 0,
     ...shard !== void 0 ? { shard } : {},
     ...shardError !== void 0 ? { shardError } : {},
+    ...configurationError !== void 0 ? { configurationError } : {},
     ...importTimeout !== void 0 ? { importTimeout } : {},
     ignored: UNSUPPORTED_VARS.filter((name) => value(env, name) !== void 0),
     warnings
@@ -547,6 +569,7 @@ function appendBatch(file, batch) {
     externalKey: batch.externalKey,
     ...batch.session !== void 0 ? { session: batch.session } : {},
     ...batch.meta !== void 0 ? { meta: batch.meta } : {},
+    ...batch.configuration !== void 0 ? { configuration: batch.configuration } : {},
     results: batch.results
   };
   fs.appendFileSync(file, `${JSON.stringify(line)}
@@ -697,8 +720,10 @@ function warnIfDropped(sent, run, log) {
 async function startRun(passed, declared) {
   const env = readEnv();
   if (env.shardError !== void 0) throw new Error(env.shardError);
+  if (env.configurationError !== void 0 && passed.configuration === void 0) throw new Error(env.configurationError);
   const stated = Object.fromEntries(Object.entries(passed).filter(([, v]) => v !== void 0));
   const cfg = { ...env.config, ...stated };
+  const configuration = cfg.configuration !== void 0 && Object.keys(cfg.configuration).length > 0 ? cfg.configuration : void 0;
   const log = cfg.log ?? ((line) => void process.stderr.write(`${line}
 `));
   const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
@@ -782,6 +807,7 @@ async function startRun(passed, declared) {
       externalKey,
       session: sessionMarker,
       ...cfg.meta !== void 0 ? { meta: cfg.meta } : {},
+      ...configuration !== void 0 ? { configuration } : {},
       results: results.map(onDisk)
     });
     stats.deferred += results.length;
@@ -806,7 +832,7 @@ async function startRun(passed, declared) {
   if (!offline) {
     if (expected !== void 0) await resolveKeys(expected.flat());
     const stated2 = expected?.slice(0, EXPECTED_MAX) ?? [];
-    const configuration = stated2.length === 0 ? void 0 : {
+    const configuration2 = stated2.length === 0 ? void 0 : {
       expected: stated2.flatMap((keys) => {
         const hit = keys.map((k) => resolved.get(k.value)).find((v) => typeof v === "string");
         if (typeof hit === "string") return [{ testCaseId: hit }];
@@ -830,7 +856,7 @@ async function startRun(passed, declared) {
       kind: cfg.kind ?? "automated",
       ...externalKey !== null ? { externalKey } : {},
       ...cfg.meta !== void 0 ? { meta: cfg.meta } : {},
-      ...configuration !== void 0 ? { configuration } : {},
+      ...configuration2 !== void 0 ? { configuration: configuration2 } : {},
       // Named even when nobody named it — the list has to read without opening a row. Only what
       // the caller DESCRIBED goes to `warnIfDropped` below: a title this side invented is not one
       // a person will miss, and a deployment older than the field must not be nagged on every run.
@@ -838,7 +864,7 @@ async function startRun(passed, declared) {
       ...described
     };
     let start = await client.post("/v1/runs", request);
-    if (!start.ok && start.status === 413 && configuration !== void 0) {
+    if (!start.ok && start.status === 413 && configuration2 !== void 0) {
       log(
         `plune: the platform refused the run's list of expected tests as too large (${start.detail || start.kind}) \u2014 starting without it: the results still land, but this run counts no tests that did not run, and claims no full run.`
       );
@@ -1384,7 +1410,8 @@ async function startRun(passed, declared) {
       }
       const out = await client.post(
         `/v1/runs/${runId}/results`,
-        { results: part }
+        // The job's combination on every batch (plune-ai/plune#1251); a refusal of it is a 422 said below as it came.
+        { results: part, ...configuration !== void 0 ? { configuration } : {} }
       );
       const counts = out.ok ? out.body?.counts : void 0;
       const told = COUNTED.filter((k) => typeof counts?.[k] === "number");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.30.2",
+  "version": "0.31.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",