@plune-ai/cli 0.2.3 → 0.4.0

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+import { z } from 'zod';
+
 interface Summary {
     total: number;
     passed: number;
@@ -37,7 +39,7 @@ interface EvalResult {
     passed: boolean;
 }
 interface RunResult {
-    schema: 1;
+    schemaVersion: 1;
     plune_version: string;
     started_at: string;
     finished_at: string;
@@ -212,4 +214,538 @@ interface RunOptions {
 }
 declare function handleRun(options: RunOptions, depsFactory?: (config: Config, baseDir: string) => RunDeps): Promise<RunResult>;
 
-export { type AssertionResultRecord, type Config, type EvalResult, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type Usage, handleRun as run };
+declare const assertionConfigSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    type: z.ZodLiteral<"exact-match">;
+    value: z.ZodString;
+    trim: z.ZodOptional<z.ZodBoolean>;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "exact-match";
+    value: string;
+    trim?: boolean | undefined;
+    ignore_case?: boolean | undefined;
+}, {
+    type: "exact-match";
+    value: string;
+    trim?: boolean | undefined;
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains">;
+    value: z.ZodString;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "contains";
+    value: string;
+    ignore_case?: boolean | undefined;
+}, {
+    type: "contains";
+    value: string;
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains-any">;
+    values: z.ZodArray<z.ZodString, "many">;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "contains-any";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}, {
+    type: "contains-any";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains-all">;
+    values: z.ZodArray<z.ZodString, "many">;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "contains-all";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}, {
+    type: "contains-all";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"json-schema">;
+    schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    extract: z.ZodOptional<z.ZodEnum<["auto", "strict"]>>;
+}, "strip", z.ZodTypeAny, {
+    type: "json-schema";
+    schema: Record<string, unknown>;
+    extract?: "auto" | "strict" | undefined;
+}, {
+    type: "json-schema";
+    schema: Record<string, unknown>;
+    extract?: "auto" | "strict" | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"llm-judge">;
+    criteria: z.ZodString;
+    provider: z.ZodOptional<z.ZodObject<{
+        type: z.ZodOptional<z.ZodEnum<["anthropic", "openai", "openrouter"]>>;
+        model: z.ZodOptional<z.ZodString>;
+        temperature: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        max_tokens: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        concurrency: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        timeout: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        max_retries: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+    }, "strip", z.ZodTypeAny, {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    }, {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    }>>;
+    pass_threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "llm-judge";
+    criteria: string;
+    provider?: {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    } | undefined;
+    pass_threshold?: number | undefined;
+}, {
+    type: "llm-judge";
+    criteria: string;
+    provider?: {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    } | undefined;
+    pass_threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"semantic-similarity">;
+    reference: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "semantic-similarity";
+    reference: string;
+    threshold?: number | undefined;
+}, {
+    type: "semantic-similarity";
+    reference: string;
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"faithfulness">;
+    context: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "faithfulness";
+    context: string;
+    threshold?: number | undefined;
+}, {
+    type: "faithfulness";
+    context: string;
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"answer-relevance">;
+    question: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "answer-relevance";
+    question: string;
+    threshold?: number | undefined;
+}, {
+    type: "answer-relevance";
+    question: string;
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"context-precision">;
+    context: z.ZodString;
+    question: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "context-precision";
+    context: string;
+    question: string;
+    threshold?: number | undefined;
+}, {
+    type: "context-precision";
+    context: string;
+    question: string;
+    threshold?: number | undefined;
+}>]>;
+
+/**
+ * Runtime validation for the RunResult contract — the JSON that `plune run --format json` emits.
+ *
+ * The canonical TypeScript shape is owned by `./results.ts` (golden-pinned, `schemaVersion: 1`,
+ * snake_case). This module is its zod mirror: the validator for anything that has to *ingest* that
+ * output rather than produce it. `plune sync` validates the local run file with it before uploading,
+ * and the Plune platform validates the same body on `POST /v1/runs` — one schema, both ends, so a
+ * divergent copy is impossible (the reason it lives here and not in the platform).
+ *
+ * Superset: an assertion record may carry an optional `binaryVerdicts[]` — the per-question breakdown
+ * of an llm-judge verdict `{ question, passed }`. The runner does not emit the field; plain output
+ * validates unchanged, and a consumer that enriches a result stays valid. Purely additive.
+ */
+/**
+ * Version marker for the RunResult contract, type-pinned to the frozen `RunResult['schemaVersion']`
+ * literal: if the public contract ever bumps its version, this line fails typecheck until it is
+ * updated in lockstep with the golden test.
+ */
+declare const RUN_RESULT_SCHEMA_VERSION: RunResult['schemaVersion'];
+declare const binaryVerdictSchema: z.ZodObject<{
+    question: z.ZodString;
+    passed: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    question: string;
+    passed: boolean;
+}, {
+    question: string;
+    passed: boolean;
+}>;
+declare const assertionResultRecordSchema: z.ZodObject<{
+    type: z.ZodString;
+    passed: z.ZodBoolean;
+    score: z.ZodOptional<z.ZodNumber>;
+    reason: z.ZodOptional<z.ZodString>;
+    binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        question: z.ZodString;
+        passed: z.ZodBoolean;
+    }, "strip", z.ZodTypeAny, {
+        question: string;
+        passed: boolean;
+    }, {
+        question: string;
+        passed: boolean;
+    }>, "many">>;
+}, "strip", z.ZodTypeAny, {
+    type: string;
+    passed: boolean;
+    score?: number | undefined;
+    reason?: string | undefined;
+    binaryVerdicts?: {
+        question: string;
+        passed: boolean;
+    }[] | undefined;
+}, {
+    type: string;
+    passed: boolean;
+    score?: number | undefined;
+    reason?: string | undefined;
+    binaryVerdicts?: {
+        question: string;
+        passed: boolean;
+    }[] | undefined;
+}>;
+declare const runResultSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    plune_version: z.ZodString;
+    started_at: z.ZodString;
+    finished_at: z.ZodString;
+    config_hash: z.ZodString;
+    summary: z.ZodObject<{
+        total: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        errored: z.ZodNumber;
+        cost_usd: z.ZodNumber;
+        duration_ms: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    }, {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    }>;
+    evals: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        tags: z.ZodArray<z.ZodString, "many">;
+        rows: z.ZodArray<z.ZodObject<{
+            vars: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            output: z.ZodNullable<z.ZodString>;
+            cached: z.ZodBoolean;
+            usage: z.ZodOptional<z.ZodObject<{
+                input_tokens: z.ZodNumber;
+                output_tokens: z.ZodNumber;
+                cost_usd: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            }, {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            }>>;
+            latency_ms: z.ZodOptional<z.ZodNumber>;
+            error: z.ZodOptional<z.ZodObject<{
+                code: z.ZodString;
+                message: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                code: string;
+                message: string;
+            }, {
+                code: string;
+                message: string;
+            }>>;
+            assertions: z.ZodArray<z.ZodObject<{
+                type: z.ZodString;
+                passed: z.ZodBoolean;
+                score: z.ZodOptional<z.ZodNumber>;
+                reason: z.ZodOptional<z.ZodString>;
+                binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    question: z.ZodString;
+                    passed: z.ZodBoolean;
+                }, "strip", z.ZodTypeAny, {
+                    question: string;
+                    passed: boolean;
+                }, {
+                    question: string;
+                    passed: boolean;
+                }>, "many">>;
+            }, "strip", z.ZodTypeAny, {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }, {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }>, "many">;
+        }, "strip", z.ZodTypeAny, {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }, {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }>, "many">;
+        passed: z.ZodBoolean;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }, {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    evals: {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }[];
+    schemaVersion: 1;
+    plune_version: string;
+    summary: {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    };
+    started_at: string;
+    finished_at: string;
+    config_hash: string;
+}, {
+    evals: {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }[];
+    schemaVersion: 1;
+    plune_version: string;
+    summary: {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    };
+    started_at: string;
+    finished_at: string;
+    config_hash: string;
+}>;
+/**
+ * Type inferred from the zod mirror — structurally a superset of the frozen `RunResult`
+ * (adds optional `binaryVerdicts`). The canonical `RunResult` remains the source of truth.
+ */
+type ParsedRunResult = z.infer<typeof runResultSchema>;
+type BinaryVerdict = z.infer<typeof binaryVerdictSchema>;
+/** Parse + validate an unknown value as a RunResult. Throws ZodError on failure. */
+declare function parseRunResult(value: unknown): ParsedRunResult;
+
+export { type AssertionResultRecord, type BinaryVerdict, type Config, type EvalResult, type ParsedRunResult, RUN_RESULT_SCHEMA_VERSION, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type Usage, assertionConfigSchema, assertionResultRecordSchema, binaryVerdictSchema, parseRunResult, handleRun as run, runResultSchema };

package/dist/index.js

@@ -1389,7 +1389,7 @@ async function runOrchestration(config, options, deps) {
     }
   }
   return {
-    schema: 1,
+    schemaVersion: 1,
     plune_version: PLUNE_VERSION,
     started_at: new Date(started).toISOString(),
     finished_at: new Date(finished).toISOString(),
@@ -1460,6 +1460,71 @@ async function handleRun(options, depsFactory) {
     deps.cache.close();
   }
 }
+
+// src/types/run-result-schema.ts
+import { z as z2 } from "zod";
+var RUN_RESULT_SCHEMA_VERSION = 1;
+var binaryVerdictSchema = z2.object({
+  question: z2.string().min(1),
+  passed: z2.boolean()
+});
+var assertionResultRecordSchema = z2.object({
+  type: z2.string(),
+  passed: z2.boolean(),
+  score: z2.number().optional(),
+  reason: z2.string().optional(),
+  binaryVerdicts: z2.array(binaryVerdictSchema).optional()
+});
+var usageSchema = z2.object({
+  input_tokens: z2.number(),
+  output_tokens: z2.number(),
+  cost_usd: z2.number()
+});
+var runErrorSchema = z2.object({
+  code: z2.string(),
+  message: z2.string()
+});
+var rowResultSchema = z2.object({
+  vars: z2.record(z2.unknown()),
+  output: z2.string().nullable(),
+  cached: z2.boolean(),
+  usage: usageSchema.optional(),
+  latency_ms: z2.number().optional(),
+  error: runErrorSchema.optional(),
+  assertions: z2.array(assertionResultRecordSchema)
+});
+var summarySchema = z2.object({
+  total: z2.number(),
+  passed: z2.number(),
+  failed: z2.number(),
+  errored: z2.number(),
+  cost_usd: z2.number(),
+  duration_ms: z2.number()
+});
+var evalResultSchema = z2.object({
+  id: z2.string(),
+  tags: z2.array(z2.string()),
+  rows: z2.array(rowResultSchema),
+  passed: z2.boolean()
+});
+var runResultSchema = z2.object({
+  schemaVersion: z2.literal(RUN_RESULT_SCHEMA_VERSION),
+  plune_version: z2.string(),
+  started_at: z2.string(),
+  finished_at: z2.string(),
+  config_hash: z2.string(),
+  summary: summarySchema,
+  evals: z2.array(evalResultSchema)
+});
+function parseRunResult(value) {
+  return runResultSchema.parse(value);
+}
 export {
-  handleRun as run
+  RUN_RESULT_SCHEMA_VERSION,
+  assertionConfigSchema,
+  assertionResultRecordSchema,
+  binaryVerdictSchema,
+  parseRunResult,
+  handleRun as run,
+  runResultSchema
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.2.3",
+  "version": "0.4.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",