@plune-ai/cli 0.17.0 → 0.18.0

package/dist/index.d.ts

@@ -5,6 +5,14 @@ interface Summary {
     passed: number;
     failed: number;
     errored: number;
+    /**
+     * plune#1245. Rows the runner said were never judged (`RowResult.inconclusive`), counted apart from
+     * `failed`. The runner always writes both; they are optional so a RunResult written before them is
+     * still valid and `schemaVersion` stays 1. With them, `passed + failed + errored + blocked + skipped`
+     * is `total`.
+     */
+    blocked?: number;
+    skipped?: number;
     cost_usd: number;
     duration_ms: number;
 }
@@ -23,6 +31,54 @@ interface RunError {
     code: string;
     message: string;
 }
+interface ManualStep {
+    action: string;
+    expected?: string;
+}
+interface ExternalKey {
+    kind: 'playwright-id' | 'path-title' | 'allure-history' | 'cairn-stable' | 'eval-id' | 'qase' | 'testrail';
+    value: string;
+}
+type ManualCase = {
+    type: 'manual';
+    steps: {
+        action: string;
+        expected?: string;
+    }[];
+    expected?: string;
+};
+type AutomatedCase = {
+    type: 'automated';
+    specRef: string;
+};
+type EvalCase = {
+    type: 'eval';
+    prompt: string;
+    goldenSetId?: string;
+    assertions: AssertionResultRecord[];
+};
+interface TestCaseBase {
+    schemaVersion: 2;
+    id: string;
+    externalKeys?: ExternalKey[];
+    is_flaky?: boolean;
+    is_muted?: boolean;
+}
+type TestCase = TestCaseBase & (ManualCase | AutomatedCase | EvalCase);
+/**
+ * The two words a row may use to say the product was never judged (ADR 0045).
+ *
+ * Both are already statuses this system has — they are `RESULT_STATUSES` entries — and neither was
+ * reachable through this contract: a row could only report `passed`, `failed` or `broken`. So a
+ * runner whose row was stopped by something outside the test had exactly one word left, `failed`,
+ * and the word was wrong. A red build because a dependency fell over reads identically to a red
+ * build because the code is broken, and those are two different pieces of work.
+ *
+ * `invalid` is deliberately absent: Qase's `invalid` is our `broken`, and a second word for one
+ * idea would fork the contract at both ends.
+ */
+declare const ROW_INCONCLUSIVE: readonly ["blocked", "skipped"];
+type RowInconclusive = (typeof ROW_INCONCLUSIVE)[number];
 interface RowResult {
     vars: Record<string, unknown>;
     output: string | null;
@@ -31,6 +87,14 @@ interface RowResult {
     latency_ms?: number;
     error?: RunError;
     assertions: AssertionResultRecord[];
+    /**
+     * ADR 0045. Say it when the test was never judged, rather than letting `failed` stand for both
+     * "the environment broke" and "the code is wrong".
+     *
+     * Optional and purely additive: a producer that does not send it validates exactly as before,
+     * which is why `schemaVersion` stays 1.
+     */
+    inconclusive?: RowInconclusive;
 }
 interface EvalResult {
     id: string;
@@ -374,6 +438,8 @@ declare const runResultSchema: z.ZodObject<{
         passed: z.ZodNumber;
         failed: z.ZodNumber;
         errored: z.ZodNumber;
+        blocked: z.ZodOptional<z.ZodNumber>;
+        skipped: z.ZodOptional<z.ZodNumber>;
         cost_usd: z.ZodNumber;
         duration_ms: z.ZodNumber;
     }, z.core.$strip>;
@@ -404,6 +470,10 @@ declare const runResultSchema: z.ZodObject<{
                     passed: z.ZodBoolean;
                 }, z.core.$strip>>>;
             }, z.core.$strip>>;
+            inconclusive: z.ZodOptional<z.ZodEnum<{
+                blocked: "blocked";
+                skipped: "skipped";
+            }>>;
         }, z.core.$strip>>;
         passed: z.ZodBoolean;
     }, z.core.$strip>>;
@@ -417,6 +487,133 @@ type BinaryVerdict = z.infer<typeof binaryVerdictSchema>;
 /** Parse + validate an unknown value as a RunResult. Throws ZodError on failure. */
 declare function parseRunResult(value: unknown): ParsedRunResult;
 
+/**
+ * Runtime validation for the TestCase contract — the JSON that `plune run import` emits
+ * or that the platform accepts on `POST /v1/test-cases`.
+ *
+ * The canonical TypeScript shape is owned by `./results.ts` (`TEST_CASE_SCHEMA_VERSION: 2`).
+ * This module is its zod mirror: the validator for anything that has to *ingest* that
+ * output rather than produce it. `plune sync` validates the local case file with it
+ * before uploading, and the Plune platform validates the same body on
+ * `POST /v1/test-cases` — one schema, both ends.
+ */
+declare const TEST_CASE_SCHEMA_VERSION: TestCase['schemaVersion'];
+declare const testCaseSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    schemaVersion: z.ZodLiteral<2>;
+    id: z.ZodString;
+    externalKeys: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        kind: z.ZodEnum<{
+            "playwright-id": "playwright-id";
+            "path-title": "path-title";
+            "allure-history": "allure-history";
+            "cairn-stable": "cairn-stable";
+            "eval-id": "eval-id";
+            qase: "qase";
+            testrail: "testrail";
+        }>;
+        value: z.ZodString;
+    }, z.core.$strip>>>;
+    is_flaky: z.ZodOptional<z.ZodBoolean>;
+    is_muted: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"manual">;
+    steps: z.ZodArray<z.ZodObject<{
+        action: z.ZodString;
+        expected: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    expected: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    schemaVersion: z.ZodLiteral<2>;
+    id: z.ZodString;
+    externalKeys: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        kind: z.ZodEnum<{
+            "playwright-id": "playwright-id";
+            "path-title": "path-title";
+            "allure-history": "allure-history";
+            "cairn-stable": "cairn-stable";
+            "eval-id": "eval-id";
+            qase: "qase";
+            testrail: "testrail";
+        }>;
+        value: z.ZodString;
+    }, z.core.$strip>>>;
+    is_flaky: z.ZodOptional<z.ZodBoolean>;
+    is_muted: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"automated">;
+    specRef: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    schemaVersion: z.ZodLiteral<2>;
+    id: z.ZodString;
+    externalKeys: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        kind: z.ZodEnum<{
+            "playwright-id": "playwright-id";
+            "path-title": "path-title";
+            "allure-history": "allure-history";
+            "cairn-stable": "cairn-stable";
+            "eval-id": "eval-id";
+            qase: "qase";
+            testrail: "testrail";
+        }>;
+        value: z.ZodString;
+    }, z.core.$strip>>>;
+    is_flaky: z.ZodOptional<z.ZodBoolean>;
+    is_muted: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"eval">;
+    prompt: z.ZodString;
+    goldenSetId: z.ZodOptional<z.ZodString>;
+    assertions: z.ZodArray<z.ZodObject<{
+        type: z.ZodString;
+        passed: z.ZodBoolean;
+        score: z.ZodOptional<z.ZodNumber>;
+        reason: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>], "type">;
+/** Parse + validate an unknown value as a TestCase. Throws ZodError on failure. */
+declare function parseTestCase(value: unknown): {
+    schemaVersion: 2;
+    id: string;
+    type: "manual";
+    steps: {
+        action: string;
+        expected?: string | undefined;
+    }[];
+    externalKeys?: {
+        kind: "playwright-id" | "path-title" | "allure-history" | "cairn-stable" | "eval-id" | "qase" | "testrail";
+        value: string;
+    }[] | undefined;
+    is_flaky?: boolean | undefined;
+    is_muted?: boolean | undefined;
+    expected?: string | undefined;
+} | {
+    schemaVersion: 2;
+    id: string;
+    type: "automated";
+    specRef: string;
+    externalKeys?: {
+        kind: "playwright-id" | "path-title" | "allure-history" | "cairn-stable" | "eval-id" | "qase" | "testrail";
+        value: string;
+    }[] | undefined;
+    is_flaky?: boolean | undefined;
+    is_muted?: boolean | undefined;
+} | {
+    schemaVersion: 2;
+    id: string;
+    type: "eval";
+    prompt: string;
+    assertions: {
+        type: string;
+        passed: boolean;
+        score?: number | undefined;
+        reason?: string | undefined;
+    }[];
+    externalKeys?: {
+        kind: "playwright-id" | "path-title" | "allure-history" | "cairn-stable" | "eval-id" | "qase" | "testrail";
+        value: string;
+    }[] | undefined;
+    is_flaky?: boolean | undefined;
+    is_muted?: boolean | undefined;
+    goldenSetId?: string | undefined;
+};
+
 interface IngestDeps {
     /** The Cairn run directory to read. Omit it and the newest run under `runsDir` is used. */
     dir?: string;
@@ -485,4 +682,4 @@ interface ReadCairnRunDeps {
  */
 declare function readCairnRun(dir: string, deps?: ReadCairnRunDeps): Promise<CairnIngestPayload>;
 
-export { type AssertionResultRecord, type BinaryVerdict, type CairnIngestCase, type CairnIngestPayload, type Config, type EvalResult, type IngestDeps, type IngestResult, NotLoggedInError, type ParsedRunResult, RUN_RESULT_SCHEMA_VERSION, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type SyncDeps, SyncFileError, SyncHttpError, SyncNetworkError, type SyncResult, TokenRejectedError, type Usage, assertionConfigSchema, assertionResultRecordSchema, binaryVerdictSchema, formatIngestResult, handleIngest as ingest, parseRunResult, readCairnRun, reportIngestFailure, reportSyncFailure, handleRun as run, runResultSchema, handleSync as sync };
+export { type AssertionResultRecord, type BinaryVerdict, type CairnIngestCase, type CairnIngestPayload, type Config, type EvalResult, type ExternalKey, type IngestDeps, type IngestResult, type ManualStep, NotLoggedInError, type ParsedRunResult, RUN_RESULT_SCHEMA_VERSION, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type SyncDeps, SyncFileError, SyncHttpError, SyncNetworkError, type SyncResult, TEST_CASE_SCHEMA_VERSION, type TestCase, TokenRejectedError, type Usage, assertionConfigSchema, assertionResultRecordSchema, binaryVerdictSchema, formatIngestResult, handleIngest as ingest, parseRunResult, parseTestCase, readCairnRun, reportIngestFailure, reportSyncFailure, handleRun as run, runResultSchema, handleSync as sync, testCaseSchema };

package/dist/index.js

@@ -1216,6 +1216,64 @@ async function mapLimit(items, limit, fn) {
   return results;
 }
 
+// src/types/test-case-schema.ts
+import { z as z2 } from "zod";
+var TEST_CASE_SCHEMA_VERSION = 2;
+var externalKeySchema = z2.object({
+  kind: z2.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
+  value: z2.string().min(1)
+});
+var manualStepSchema = z2.object({
+  action: z2.string().min(1),
+  expected: z2.string().min(1).optional()
+});
+var assertionResultRecordSchema = z2.object({
+  type: z2.string().min(1),
+  passed: z2.boolean(),
+  score: z2.number().optional(),
+  reason: z2.string().optional()
+});
+var manualCaseSchema = z2.object({
+  type: z2.literal("manual"),
+  steps: z2.array(manualStepSchema).min(1),
+  expected: z2.string().min(1).optional()
+});
+var automatedCaseSchema = z2.object({
+  type: z2.literal("automated"),
+  specRef: z2.string().min(1)
+});
+var evalCaseSchema = z2.object({
+  type: z2.literal("eval"),
+  prompt: z2.string().min(1),
+  goldenSetId: z2.string().min(1).optional(),
+  assertions: z2.array(assertionResultRecordSchema).min(1)
+});
+var testCaseBaseSchema = z2.object({
+  schemaVersion: z2.literal(TEST_CASE_SCHEMA_VERSION),
+  id: z2.string().min(1),
+  externalKeys: z2.array(externalKeySchema).optional(),
+  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+  is_flaky: z2.boolean().optional(),
+  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+  is_muted: z2.boolean().optional()
+});
+var testCaseSchema = z2.discriminatedUnion("type", [
+  testCaseBaseSchema.merge(manualCaseSchema),
+  testCaseBaseSchema.merge(automatedCaseSchema),
+  testCaseBaseSchema.merge(evalCaseSchema)
+]);
+function parseTestCase(value) {
+  return testCaseSchema.parse(value);
+}
+
+// src/types/results.ts
+var ROW_INCONCLUSIVE = ["blocked", "skipped"];
+function rowStatus(row) {
+  if (row.inconclusive !== void 0) return row.inconclusive;
+  if (row.error !== void 0) return "errored";
+  return row.assertions.every((a) => a.passed) ? "passed" : "failed";
+}
+
 // src/orchestrator/run.ts
 var DEFAULT_MAX_TOKENS2 = 1024;
 var PLUNE_VERSION = "0.1.0";
@@ -1358,10 +1416,25 @@ function resolveTemplate(ev, baseDir) {
   }
   throw new RunConfigError(`Eval "${ev.id}" has neither prompt nor prompt_file`);
 }
-function classify(row) {
-  if (row.error !== void 0) return "errored";
-  if (row.assertions.some((a) => !a.passed)) return "failed";
-  return "passed";
+function summarize(evals, durationMs) {
+  const summary = {
+    total: 0,
+    passed: 0,
+    failed: 0,
+    errored: 0,
+    blocked: 0,
+    skipped: 0,
+    cost_usd: 0,
+    duration_ms: durationMs
+  };
+  for (const ev of evals) {
+    for (const row of ev.rows) {
+      summary.total += 1;
+      summary[rowStatus(row)] += 1;
+      summary.cost_usd += row.usage?.cost_usd ?? 0;
+    }
+  }
+  return summary;
 }
 async function runOrchestration(config, options, deps) {
   const started = deps.now();
@@ -1396,26 +1469,12 @@ async function runOrchestration(config, options, deps) {
         noCache
       })
     );
-    const passed = rowResults.every((r) => classify(r) === "passed");
+    const passed = rowResults.every((r) => rowStatus(r) === "passed");
     evalResults.push({ id: ev.id, tags: ev.tags ?? [], rows: rowResults, passed });
     if (options.bail === true && !passed) break;
   }
   const finished = deps.now();
-  const summary = {
-    total: 0,
-    passed: 0,
-    failed: 0,
-    errored: 0,
-    cost_usd: 0,
-    duration_ms: finished - started
-  };
-  for (const ev of evalResults) {
-    for (const row of ev.rows) {
-      summary.total += 1;
-      summary[classify(row)] += 1;
-      summary.cost_usd += row.usage?.cost_usd ?? 0;
-    }
-  }
+  const summary = summarize(evalResults, finished - started);
   return {
     schemaVersion: 1,
     plune_version: PLUNE_VERSION,
@@ -1499,59 +1558,64 @@ import * as fs6 from "fs";
 import * as path5 from "path";
 
 // src/types/run-result-schema.ts
-import { z as z2 } from "zod";
+import { z as z3 } from "zod";
 var RUN_RESULT_SCHEMA_VERSION = 1;
-var binaryVerdictSchema = z2.object({
-  question: z2.string().min(1),
-  passed: z2.boolean()
+var binaryVerdictSchema = z3.object({
+  question: z3.string().min(1),
+  passed: z3.boolean()
 });
-var assertionResultRecordSchema = z2.object({
-  type: z2.string(),
-  passed: z2.boolean(),
-  score: z2.number().optional(),
-  reason: z2.string().optional(),
-  binaryVerdicts: z2.array(binaryVerdictSchema).optional()
+var assertionResultRecordSchema2 = z3.object({
+  type: z3.string(),
+  passed: z3.boolean(),
+  score: z3.number().optional(),
+  reason: z3.string().optional(),
+  binaryVerdicts: z3.array(binaryVerdictSchema).optional()
 });
-var usageSchema = z2.object({
-  input_tokens: z2.number(),
-  output_tokens: z2.number(),
-  cost_usd: z2.number()
+var usageSchema = z3.object({
+  input_tokens: z3.number(),
+  output_tokens: z3.number(),
+  cost_usd: z3.number()
 });
-var runErrorSchema = z2.object({
-  code: z2.string(),
-  message: z2.string()
+var runErrorSchema = z3.object({
+  code: z3.string(),
+  message: z3.string()
 });
-var rowResultSchema = z2.object({
-  vars: z2.record(z2.string(), z2.unknown()),
-  output: z2.string().nullable(),
-  cached: z2.boolean(),
+var rowResultSchema = z3.object({
+  vars: z3.record(z3.string(), z3.unknown()),
+  output: z3.string().nullable(),
+  cached: z3.boolean(),
   usage: usageSchema.optional(),
-  latency_ms: z2.number().optional(),
+  latency_ms: z3.number().optional(),
   error: runErrorSchema.optional(),
-  assertions: z2.array(assertionResultRecordSchema)
+  assertions: z3.array(assertionResultRecordSchema2),
+  /** ADR 0045. Optional and additive — the same enum the platform validates on `POST /v1/runs`. */
+  inconclusive: z3.enum(ROW_INCONCLUSIVE).optional()
 });
-var summarySchema = z2.object({
-  total: z2.number(),
-  passed: z2.number(),
-  failed: z2.number(),
-  errored: z2.number(),
-  cost_usd: z2.number(),
-  duration_ms: z2.number()
+var summarySchema = z3.object({
+  total: z3.number(),
+  passed: z3.number(),
+  failed: z3.number(),
+  errored: z3.number(),
+  /** plune#1245. Optional and additive — a RunResult written before them validates unchanged. */
+  blocked: z3.number().optional(),
+  skipped: z3.number().optional(),
+  cost_usd: z3.number(),
+  duration_ms: z3.number()
 });
-var evalResultSchema = z2.object({
-  id: z2.string(),
-  tags: z2.array(z2.string()),
-  rows: z2.array(rowResultSchema),
-  passed: z2.boolean()
+var evalResultSchema = z3.object({
+  id: z3.string(),
+  tags: z3.array(z3.string()),
+  rows: z3.array(rowResultSchema),
+  passed: z3.boolean()
 });
-var runResultSchema = z2.object({
-  schemaVersion: z2.literal(RUN_RESULT_SCHEMA_VERSION),
-  plune_version: z2.string(),
-  started_at: z2.string(),
-  finished_at: z2.string(),
-  config_hash: z2.string(),
+var runResultSchema = z3.object({
+  schemaVersion: z3.literal(RUN_RESULT_SCHEMA_VERSION),
+  plune_version: z3.string(),
+  started_at: z3.string(),
+  finished_at: z3.string(),
+  config_hash: z3.string(),
   summary: summarySchema,
-  evals: z2.array(evalResultSchema)
+  evals: z3.array(evalResultSchema)
 });
 function parseRunResult(value) {
   return runResultSchema.parse(value);
@@ -1934,17 +1998,20 @@ export {
   SyncFileError,
   SyncHttpError,
   SyncNetworkError,
+  TEST_CASE_SCHEMA_VERSION,
   TokenRejectedError,
   assertionConfigSchema,
-  assertionResultRecordSchema,
+  assertionResultRecordSchema2 as assertionResultRecordSchema,
   binaryVerdictSchema,
   formatIngestResult,
   handleIngest as ingest,
   parseRunResult,
+  parseTestCase,
   readCairnRun,
   reportIngestFailure,
   reportSyncFailure,
   handleRun as run,
   runResultSchema,
-  handleSync as sync
+  handleSync as sync,
+  testCaseSchema
 };