@plune-ai/cli 0.34.2 → 0.34.4

package/README.md

@@ -70,8 +70,10 @@ evals:
         value: "Paris"
 ```
 
-Datasets are JSONL, one row per line, shaped `{ "vars": { ... }, "expected"?: "..." }`. The
-provider API key is read from the environment based on `provider.type`:
+Datasets are JSONL, one row per line, shaped `{ "vars": { ... }, "expected"?: "..." }`. A line that
+is not valid JSON or not a row (no `vars`, or `vars` that is not an object) stops the run before
+any model call with `<file>:<line>` and what is wrong, exit code `2`, like any other configuration
+error. The provider API key is read from the environment based on `provider.type`:
 
 | Provider   | `provider.type` | Environment variable |
 | ---------- | --------------- | -------------------- |

package/dist/cli.cjs

@@ -2096,7 +2096,31 @@ function loadDataset(ref, baseDir) {
     return ref.examples;
   }
   const content = fs4.readFileSync(path4.resolve(baseDir, ref), "utf8");
-  return content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+  const rows = [];
+  content.split(/\r?\n/).forEach((line, i) => {
+    if (line.trim().length === 0) return;
+    let row;
+    try {
+      row = JSON.parse(line.trim());
+    } catch (err) {
+      throw new RunConfigError(`${ref}:${i + 1}: not valid JSON (${err.message})`);
+    }
+    const vars = isObject(row) ? row["vars"] : void 0;
+    if (!isObject(vars)) {
+      const why = isObject(row) && "vars" in row ? `vars: Invalid input: expected record, received ${typeName(vars)}` : 'this row has no "vars"';
+      throw new RunConfigError(
+        `${ref}:${i + 1}: a dataset row must be { "vars": { ... } }; ${why}`
+      );
+    }
+    rows.push(row);
+  });
+  return rows;
+}
+function isObject(value2) {
+  return typeof value2 === "object" && value2 !== null && !Array.isArray(value2);
+}
+function typeName(value2) {
+  return value2 === null ? "null" : Array.isArray(value2) ? "array" : typeof value2;
 }
 var fs4, path4;
 var init_dataset = __esm({
@@ -2105,6 +2129,7 @@ var init_dataset = __esm({
     init_cjs_shims();
     fs4 = __toESM(require("fs"), 1);
     path4 = __toESM(require("path"), 1);
+    init_run();
   }
 });
 
@@ -5662,21 +5687,21 @@ var init_playwright_json = __esm({
 });
 
 // src/importers/k6.ts
-function isObject(value2) {
+function isObject2(value2) {
   return typeof value2 === "object" && value2 !== null && !Array.isArray(value2);
 }
 function isK6Summary(parsed) {
-  if (!isObject(parsed) || !isObject(parsed["metrics"])) return false;
-  if (isObject(parsed["root_group"])) return true;
-  return Object.values(parsed["metrics"]).some((m) => isObject(m) && (typeof m["type"] === "string" || isObject(m["thresholds"])));
+  if (!isObject2(parsed) || !isObject2(parsed["metrics"])) return false;
+  if (isObject2(parsed["root_group"])) return true;
+  return Object.values(parsed["metrics"]).some((m) => isObject2(m) && (typeof m["type"] === "string" || isObject2(m["thresholds"])));
 }
 function passedOf(verdict) {
-  if (isObject(verdict) && typeof verdict["ok"] === "boolean") return verdict["ok"];
+  if (isObject2(verdict) && typeof verdict["ok"] === "boolean") return verdict["ok"];
   if (typeof verdict === "boolean") return !verdict;
   return void 0;
 }
 function measuredOf(metric) {
-  const values = isObject(metric.values) ? metric.values : metric;
+  const values = isObject2(metric.values) ? metric.values : metric;
   const unit = metric.contains === "time" ? " ms" : "";
   return Object.entries(values).filter(([name, v]) => name !== "thresholds" && typeof v === "number").map(([name, v]) => `${name}=${Math.round(v * 100) / 100}${unit}`).join(", ");
 }
@@ -5687,13 +5712,13 @@ function readK6(source, file) {
   } catch (err) {
     throw new JsonReportError(file, `not JSON \u2014 ${printable(err instanceof Error ? err.message : String(err))}`);
   }
-  if (!isObject(parsed) || !isObject(parsed["metrics"])) {
+  if (!isObject2(parsed) || !isObject2(parsed["metrics"])) {
     throw new JsonReportError(file, 'no "metrics" object \u2014 is this a k6 summary (k6 run --summary-export, or the data handleSummary receives)?');
   }
   const specRef = path13.basename(file);
   const out = [];
   for (const [name, raw] of Object.entries(parsed["metrics"])) {
-    if (!isObject(raw) || !isObject(raw["thresholds"])) continue;
+    if (!isObject2(raw) || !isObject2(raw["thresholds"])) continue;
     const metric = raw;
     const measured = measuredOf(metric);
     for (const [expression, verdict] of Object.entries(metric.thresholds)) {
@@ -5734,11 +5759,11 @@ var init_k6 = __esm({
 });
 
 // src/importers/lighthouse.ts
-function isObject2(value2) {
+function isObject3(value2) {
   return typeof value2 === "object" && value2 !== null && !Array.isArray(value2);
 }
 function isLighthouseReport(parsed) {
-  return isObject2(parsed) && typeof parsed["lighthouseVersion"] === "string" && isObject2(parsed["categories"]);
+  return isObject3(parsed) && typeof parsed["lighthouseVersion"] === "string" && isObject3(parsed["categories"]);
 }
 function idOf(key, category) {
   return typeof category.id === "string" && category.id !== "" ? category.id : key;
@@ -5747,7 +5772,7 @@ function executionOf2(report) {
   const started = typeof report["fetchTime"] === "string" ? new Date(report["fetchTime"]) : void 0;
   if (started === void 0 || Number.isNaN(started.getTime())) return void 0;
   const timing = report["timing"];
-  const total = isObject2(timing) && typeof timing["total"] === "number" && timing["total"] >= 0 ? Math.round(timing["total"]) : 0;
+  const total = isObject3(timing) && typeof timing["total"] === "number" && timing["total"] >= 0 ? Math.round(timing["total"]) : 0;
   return {
     startedAt: started.toISOString(),
     finishedAt: new Date(started.getTime() + total).toISOString(),
@@ -5757,7 +5782,7 @@ function executionOf2(report) {
 }
 function runtimeErrorOf(report) {
   const err = report["runtimeError"];
-  if (!isObject2(err)) return void 0;
+  if (!isObject3(err)) return void 0;
   const code = typeof err["code"] === "string" ? err["code"] : void 0;
   if (code === "NO_ERROR") return void 0;
   const message = typeof err["message"] === "string" ? err["message"] : void 0;
@@ -5778,11 +5803,11 @@ function readLighthouse(source, file, minScores = {}) {
   const categories = report["categories"];
   const url = ["requestedUrl", "finalDisplayedUrl", "finalUrl"].map((k) => report[k]).find((u) => typeof u === "string" && u !== "");
   const settings = report["configSettings"];
-  const formFactor = isObject2(settings) && typeof settings["formFactor"] === "string" && settings["formFactor"] !== "" ? settings["formFactor"] : void 0;
+  const formFactor = isObject3(settings) && typeof settings["formFactor"] === "string" && settings["formFactor"] !== "" ? settings["formFactor"] : void 0;
   const runtimeError = runtimeErrorOf(report);
   const execution = executionOf2(report);
   const specRef = path14.basename(file);
-  const known = Object.entries(categories).filter(([, raw]) => isObject2(raw)).map(([key, raw]) => idOf(key, raw));
+  const known = Object.entries(categories).filter(([, raw]) => isObject3(raw)).map(([key, raw]) => idOf(key, raw));
   if (known.length > 0) {
     for (const [id2, score] of Object.entries(minScores)) {
       if (!known.includes(id2)) {
@@ -5809,7 +5834,7 @@ function readLighthouse(source, file, minScores = {}) {
     });
   };
   for (const [key, raw] of Object.entries(categories)) {
-    if (!isObject2(raw)) continue;
+    if (!isObject3(raw)) continue;
     const category = raw;
     const id2 = idOf(key, category);
     const name = typeof category.title === "string" && category.title !== "" ? category.title : id2;
@@ -6235,15 +6260,16 @@ function evalStatus(e) {
   const statuses = e.rows.map(rowStatus);
   if (statuses.includes("failed")) return "failed";
   if (statuses.includes("errored")) return "errored";
-  return "passed";
+  return statuses.includes("passed") ? "passed" : "unjudged";
 }
 function classify2(baseline, current) {
   if (current === "errored") return "errored";
+  if (current === "unjudged") return "unmeasured";
   if (baseline === "absent") return current === "passed" ? "new-pass" : "new-fail";
   if (current === "absent") return "removed";
   if (baseline === "passed") return current === "passed" ? "stable-pass" : "regression";
   if (current === "passed") return "improvement";
-  return baseline === "errored" ? "unmeasured" : "pre-existing-fail";
+  return baseline === "failed" ? "pre-existing-fail" : "unmeasured";
 }
 function diffRuns(baseline, current) {
   const baseMap = new Map(baseline.evals.map((e) => [e.id, evalStatus(e)]));
@@ -6402,7 +6428,7 @@ var init_diff3 = __esm({
       "new-pass": "\u{1F7E2} new pass",
       "stable-pass": "\u2705 stable",
       errored: "\u26A0\uFE0F errored",
-      unmeasured: "\u26A0\uFE0F not measured (baseline errored)",
+      unmeasured: "\u26A0\uFE0F not measured",
       removed: "\u2796 removed"
     };
   }
@@ -6983,7 +7009,7 @@ var init_contract = __esm({
       define({
         name: "plune_review_decide",
         title: "Decide queue entries",
-        description: "Approve or reject up to 100 review queue entries in one decision. The first call changes nothing and returns a preview with a confirm value; repeat the same call with confirm to apply it.",
+        description: "Approve or reject up to 100 review queue entries in one decision. The first call changes nothing and returns a preview with a confirm value; repeat the same call with confirm to apply it. An entry skipped as policy_refused was refused by the review policy of the project (for example, the person who proposed it cannot approve it): do not retry it \u2014 it needs another reviewer; its detail says why.",
         input: import_zod5.z.strictObject({
           itemIds: import_zod5.z.array(import_zod5.z.string().min(1)).min(1).max(BATCH_MAX2).refine((a) => new Set(a).size === a.length, { message: "an id is named once" }),
           verdict: import_zod5.z.enum(["approved", "rejected"]),

package/dist/index.cjs

@@ -1686,7 +1686,31 @@ function loadDataset(ref, baseDir) {
     return ref.examples;
   }
   const content = fs4.readFileSync(path3.resolve(baseDir, ref), "utf8");
-  return content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+  const rows = [];
+  content.split(/\r?\n/).forEach((line, i) => {
+    if (line.trim().length === 0) return;
+    let row;
+    try {
+      row = JSON.parse(line.trim());
+    } catch (err) {
+      throw new RunConfigError(`${ref}:${i + 1}: not valid JSON (${err.message})`);
+    }
+    const vars = isObject(row) ? row["vars"] : void 0;
+    if (!isObject(vars)) {
+      const why = isObject(row) && "vars" in row ? `vars: Invalid input: expected record, received ${typeName(vars)}` : 'this row has no "vars"';
+      throw new RunConfigError(
+        `${ref}:${i + 1}: a dataset row must be { "vars": { ... } }; ${why}`
+      );
+    }
+    rows.push(row);
+  });
+  return rows;
+}
+function isObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function typeName(value) {
+  return value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
 }
 
 // src/cli/commands/run.ts

package/dist/index.js

@@ -1626,7 +1626,31 @@ function loadDataset(ref, baseDir) {
     return ref.examples;
   }
   const content = fs4.readFileSync(path3.resolve(baseDir, ref), "utf8");
-  return content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+  const rows = [];
+  content.split(/\r?\n/).forEach((line, i) => {
+    if (line.trim().length === 0) return;
+    let row;
+    try {
+      row = JSON.parse(line.trim());
+    } catch (err) {
+      throw new RunConfigError(`${ref}:${i + 1}: not valid JSON (${err.message})`);
+    }
+    const vars = isObject(row) ? row["vars"] : void 0;
+    if (!isObject(vars)) {
+      const why = isObject(row) && "vars" in row ? `vars: Invalid input: expected record, received ${typeName(vars)}` : 'this row has no "vars"';
+      throw new RunConfigError(
+        `${ref}:${i + 1}: a dataset row must be { "vars": { ... } }; ${why}`
+      );
+    }
+    rows.push(row);
+  });
+  return rows;
+}
+function isObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function typeName(value) {
+  return value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
 }
 
 // src/cli/commands/run.ts

package/dist/mcp-tools.cjs

@@ -298,7 +298,7 @@ var PLUNE_TOOLS = [
   define({
     name: "plune_review_decide",
     title: "Decide queue entries",
-    description: "Approve or reject up to 100 review queue entries in one decision. The first call changes nothing and returns a preview with a confirm value; repeat the same call with confirm to apply it.",
+    description: "Approve or reject up to 100 review queue entries in one decision. The first call changes nothing and returns a preview with a confirm value; repeat the same call with confirm to apply it. An entry skipped as policy_refused was refused by the review policy of the project (for example, the person who proposed it cannot approve it): do not retry it \u2014 it needs another reviewer; its detail says why.",
     input: import_zod2.z.strictObject({
       itemIds: import_zod2.z.array(import_zod2.z.string().min(1)).min(1).max(BATCH_MAX).refine((a) => new Set(a).size === a.length, { message: "an id is named once" }),
       verdict: import_zod2.z.enum(["approved", "rejected"]),

package/dist/mcp-tools.js

@@ -262,7 +262,7 @@ var PLUNE_TOOLS = [
   define({
     name: "plune_review_decide",
     title: "Decide queue entries",
-    description: "Approve or reject up to 100 review queue entries in one decision. The first call changes nothing and returns a preview with a confirm value; repeat the same call with confirm to apply it.",
+    description: "Approve or reject up to 100 review queue entries in one decision. The first call changes nothing and returns a preview with a confirm value; repeat the same call with confirm to apply it. An entry skipped as policy_refused was refused by the review policy of the project (for example, the person who proposed it cannot approve it): do not retry it \u2014 it needs another reviewer; its detail says why.",
     input: z2.strictObject({
       itemIds: z2.array(z2.string().min(1)).min(1).max(BATCH_MAX).refine((a) => new Set(a).size === a.length, { message: "an id is named once" }),
       verdict: z2.enum(["approved", "rejected"]),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.34.2",
+  "version": "0.34.4",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",