@plune-ai/cli 0.3.0 → 0.5.0

package/dist/index.d.cts

@@ -1,3 +1,5 @@
+import { z } from 'zod';
+
 interface Summary {
     total: number;
     passed: number;
@@ -212,4 +214,589 @@ interface RunOptions {
 }
 declare function handleRun(options: RunOptions, depsFactory?: (config: Config, baseDir: string) => RunDeps): Promise<RunResult>;
 
-export { type AssertionResultRecord, type Config, type EvalResult, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type Usage, handleRun as run };
+/** No saved token — the user must `plune login` before syncing. (exit 2) */
+declare class NotLoggedInError extends Error {
+    constructor();
+}
+/** The token was present but the server rejected it (401 — revoked or invalid). (exit 2) */
+declare class TokenRejectedError extends Error {
+    constructor();
+}
+/** No run to sync, or the run file is unreadable / not a valid RunResult. (exit 2) */
+declare class SyncFileError extends Error {
+    constructor(message: string);
+}
+/** The API was unreachable (DNS / TLS / offline). Message names the host, never the token. (exit 1) */
+declare class SyncNetworkError extends Error {
+    constructor(apiUrl: string);
+}
+/** The server answered with a non-2xx that is not a 401 (e.g. 400 invalid body, 5xx). (exit 1) */
+declare class SyncHttpError extends Error {
+    readonly status: number;
+    constructor(status: number, detail: string);
+}
+interface SyncDeps {
+    /** Base dir the default run file is read from (config dir or cwd). Defaults to `process.cwd()`. */
+    cwd?: string;
+    /** Explicit run JSON to sync instead of `<cwd>/.plune/last-run.json`. */
+    file?: string;
+    /** Override the API base URL. Falls back to `PLUNE_API_URL`, then the beta default. */
+    apiUrl?: string;
+    /** Token source (injected in tests). Defaults to the real credentials store. */
+    loadToken?: () => string | null;
+    /** HTTP transport (injected in tests — e.g. a real Hono app's `request`). Defaults to global `fetch`. */
+    fetchImpl?: typeof fetch;
+}
+interface SyncResult {
+    /** Server-assigned run id. */
+    id: string;
+    /** The `GET /v1/runs/:id` URL where the uploaded run can be read back. */
+    url: string;
+}
+/**
+ * Upload the latest (or `--file`) local run to `POST {apiUrl}/v1/runs` with the stored Bearer token.
+ * Returns the server-assigned id + the read-back URL. Throws a typed error (see the classes above) on
+ * every failure path so the CLI can exit cleanly without ever leaking the token or a stack trace.
+ */
+declare function handleSync(deps?: SyncDeps): Promise<SyncResult>;
+/**
+ * Map a `handleSync` rejection to a friendly stderr line + the exit code to use, or `null` if the error
+ * is not a known sync failure (caller then treats it as unexpected).
+ */
+declare function reportSyncFailure(err: unknown, write: (s: string) => void): number | null;
+
+declare const assertionConfigSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    type: z.ZodLiteral<"exact-match">;
+    value: z.ZodString;
+    trim: z.ZodOptional<z.ZodBoolean>;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "exact-match";
+    value: string;
+    trim?: boolean | undefined;
+    ignore_case?: boolean | undefined;
+}, {
+    type: "exact-match";
+    value: string;
+    trim?: boolean | undefined;
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains">;
+    value: z.ZodString;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "contains";
+    value: string;
+    ignore_case?: boolean | undefined;
+}, {
+    type: "contains";
+    value: string;
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains-any">;
+    values: z.ZodArray<z.ZodString, "many">;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "contains-any";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}, {
+    type: "contains-any";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"contains-all">;
+    values: z.ZodArray<z.ZodString, "many">;
+    ignore_case: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    type: "contains-all";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}, {
+    type: "contains-all";
+    values: string[];
+    ignore_case?: boolean | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"json-schema">;
+    schema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    extract: z.ZodOptional<z.ZodEnum<["auto", "strict"]>>;
+}, "strip", z.ZodTypeAny, {
+    type: "json-schema";
+    schema: Record<string, unknown>;
+    extract?: "auto" | "strict" | undefined;
+}, {
+    type: "json-schema";
+    schema: Record<string, unknown>;
+    extract?: "auto" | "strict" | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"llm-judge">;
+    criteria: z.ZodString;
+    provider: z.ZodOptional<z.ZodObject<{
+        type: z.ZodOptional<z.ZodEnum<["anthropic", "openai", "openrouter"]>>;
+        model: z.ZodOptional<z.ZodString>;
+        temperature: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        max_tokens: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        concurrency: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        timeout: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        max_retries: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+    }, "strip", z.ZodTypeAny, {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    }, {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    }>>;
+    pass_threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "llm-judge";
+    criteria: string;
+    provider?: {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    } | undefined;
+    pass_threshold?: number | undefined;
+}, {
+    type: "llm-judge";
+    criteria: string;
+    provider?: {
+        type?: "anthropic" | "openai" | "openrouter" | undefined;
+        model?: string | undefined;
+        temperature?: number | undefined;
+        max_tokens?: number | undefined;
+        concurrency?: number | undefined;
+        timeout?: number | undefined;
+        max_retries?: number | undefined;
+    } | undefined;
+    pass_threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"semantic-similarity">;
+    reference: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "semantic-similarity";
+    reference: string;
+    threshold?: number | undefined;
+}, {
+    type: "semantic-similarity";
+    reference: string;
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"faithfulness">;
+    context: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "faithfulness";
+    context: string;
+    threshold?: number | undefined;
+}, {
+    type: "faithfulness";
+    context: string;
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"answer-relevance">;
+    question: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "answer-relevance";
+    question: string;
+    threshold?: number | undefined;
+}, {
+    type: "answer-relevance";
+    question: string;
+    threshold?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"context-precision">;
+    context: z.ZodString;
+    question: z.ZodString;
+    threshold: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    type: "context-precision";
+    context: string;
+    question: string;
+    threshold?: number | undefined;
+}, {
+    type: "context-precision";
+    context: string;
+    question: string;
+    threshold?: number | undefined;
+}>]>;
+
+/**
+ * Runtime validation for the RunResult contract — the JSON that `plune run --format json` emits.
+ *
+ * The canonical TypeScript shape is owned by `./results.ts` (golden-pinned, `schemaVersion: 1`,
+ * snake_case). This module is its zod mirror: the validator for anything that has to *ingest* that
+ * output rather than produce it. `plune sync` validates the local run file with it before uploading,
+ * and the Plune platform validates the same body on `POST /v1/runs` — one schema, both ends, so a
+ * divergent copy is impossible (the reason it lives here and not in the platform).
+ *
+ * Superset: an assertion record may carry an optional `binaryVerdicts[]` — the per-question breakdown
+ * of an llm-judge verdict `{ question, passed }`. The runner does not emit the field; plain output
+ * validates unchanged, and a consumer that enriches a result stays valid. Purely additive.
+ */
+/**
+ * Version marker for the RunResult contract, type-pinned to the frozen `RunResult['schemaVersion']`
+ * literal: if the public contract ever bumps its version, this line fails typecheck until it is
+ * updated in lockstep with the golden test.
+ */
+declare const RUN_RESULT_SCHEMA_VERSION: RunResult['schemaVersion'];
+declare const binaryVerdictSchema: z.ZodObject<{
+    question: z.ZodString;
+    passed: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    question: string;
+    passed: boolean;
+}, {
+    question: string;
+    passed: boolean;
+}>;
+declare const assertionResultRecordSchema: z.ZodObject<{
+    type: z.ZodString;
+    passed: z.ZodBoolean;
+    score: z.ZodOptional<z.ZodNumber>;
+    reason: z.ZodOptional<z.ZodString>;
+    binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        question: z.ZodString;
+        passed: z.ZodBoolean;
+    }, "strip", z.ZodTypeAny, {
+        question: string;
+        passed: boolean;
+    }, {
+        question: string;
+        passed: boolean;
+    }>, "many">>;
+}, "strip", z.ZodTypeAny, {
+    type: string;
+    passed: boolean;
+    score?: number | undefined;
+    reason?: string | undefined;
+    binaryVerdicts?: {
+        question: string;
+        passed: boolean;
+    }[] | undefined;
+}, {
+    type: string;
+    passed: boolean;
+    score?: number | undefined;
+    reason?: string | undefined;
+    binaryVerdicts?: {
+        question: string;
+        passed: boolean;
+    }[] | undefined;
+}>;
+declare const runResultSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    plune_version: z.ZodString;
+    started_at: z.ZodString;
+    finished_at: z.ZodString;
+    config_hash: z.ZodString;
+    summary: z.ZodObject<{
+        total: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        errored: z.ZodNumber;
+        cost_usd: z.ZodNumber;
+        duration_ms: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    }, {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    }>;
+    evals: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        tags: z.ZodArray<z.ZodString, "many">;
+        rows: z.ZodArray<z.ZodObject<{
+            vars: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            output: z.ZodNullable<z.ZodString>;
+            cached: z.ZodBoolean;
+            usage: z.ZodOptional<z.ZodObject<{
+                input_tokens: z.ZodNumber;
+                output_tokens: z.ZodNumber;
+                cost_usd: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            }, {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            }>>;
+            latency_ms: z.ZodOptional<z.ZodNumber>;
+            error: z.ZodOptional<z.ZodObject<{
+                code: z.ZodString;
+                message: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                code: string;
+                message: string;
+            }, {
+                code: string;
+                message: string;
+            }>>;
+            assertions: z.ZodArray<z.ZodObject<{
+                type: z.ZodString;
+                passed: z.ZodBoolean;
+                score: z.ZodOptional<z.ZodNumber>;
+                reason: z.ZodOptional<z.ZodString>;
+                binaryVerdicts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    question: z.ZodString;
+                    passed: z.ZodBoolean;
+                }, "strip", z.ZodTypeAny, {
+                    question: string;
+                    passed: boolean;
+                }, {
+                    question: string;
+                    passed: boolean;
+                }>, "many">>;
+            }, "strip", z.ZodTypeAny, {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }, {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }>, "many">;
+        }, "strip", z.ZodTypeAny, {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }, {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }>, "many">;
+        passed: z.ZodBoolean;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }, {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    evals: {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }[];
+    schemaVersion: 1;
+    plune_version: string;
+    summary: {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    };
+    started_at: string;
+    finished_at: string;
+    config_hash: string;
+}, {
+    evals: {
+        id: string;
+        tags: string[];
+        passed: boolean;
+        rows: {
+            vars: Record<string, unknown>;
+            assertions: {
+                type: string;
+                passed: boolean;
+                score?: number | undefined;
+                reason?: string | undefined;
+                binaryVerdicts?: {
+                    question: string;
+                    passed: boolean;
+                }[] | undefined;
+            }[];
+            output: string | null;
+            cached: boolean;
+            error?: {
+                code: string;
+                message: string;
+            } | undefined;
+            usage?: {
+                cost_usd: number;
+                input_tokens: number;
+                output_tokens: number;
+            } | undefined;
+            latency_ms?: number | undefined;
+        }[];
+    }[];
+    schemaVersion: 1;
+    plune_version: string;
+    summary: {
+        cost_usd: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        total: number;
+        duration_ms: number;
+    };
+    started_at: string;
+    finished_at: string;
+    config_hash: string;
+}>;
+/**
+ * Type inferred from the zod mirror — structurally a superset of the frozen `RunResult`
+ * (adds optional `binaryVerdicts`). The canonical `RunResult` remains the source of truth.
+ */
+type ParsedRunResult = z.infer<typeof runResultSchema>;
+type BinaryVerdict = z.infer<typeof binaryVerdictSchema>;
+/** Parse + validate an unknown value as a RunResult. Throws ZodError on failure. */
+declare function parseRunResult(value: unknown): ParsedRunResult;
+
+export { type AssertionResultRecord, type BinaryVerdict, type Config, type EvalResult, NotLoggedInError, type ParsedRunResult, RUN_RESULT_SCHEMA_VERSION, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type SyncDeps, SyncFileError, SyncHttpError, SyncNetworkError, type SyncResult, TokenRejectedError, type Usage, assertionConfigSchema, assertionResultRecordSchema, binaryVerdictSchema, parseRunResult, reportSyncFailure, handleRun as run, runResultSchema, handleSync as sync };