@opencode_weave/weave 0.7.0 → 0.7.3

package/dist/agents/agent-builder.d.ts

@@ -8,12 +8,34 @@ export type BuildAgentOptions = {
     resolveSkills?: ResolveSkillsFn;
     disabledAgents?: Set<string>;
 };
+/**
+ * Map from agent config key (lowercase) to display name variants that
+ * might appear in prompt text. Used by stripDisabledAgentReferences to
+ * remove lines that mention disabled agents.
+ *
+ * Exported for test cleanup — tests that call addBuiltinNameVariant
+ * must restore original arrays in afterEach to avoid state pollution.
+ */
+export declare const AGENT_NAME_VARIANTS: Record<string, string[]>;
+/**
+ * Reset the mutable name variants map to its initial state.
+ * Used by tests to prevent cross-test state pollution.
+ */
+export declare function resetNameVariants(): void;
 /**
  * Register name variants for a custom agent so that
  * `stripDisabledAgentReferences` can strip its references from prompts.
  * Does not override existing (builtin) entries.
  */
 export declare function registerAgentNameVariants(name: string, variants?: string[]): void;
+/**
+ * Add additional name variants for a builtin agent.
+ * Used when a user sets a custom display_name — the custom name
+ * must be included in variants so stripDisabledAgentReferences
+ * can match it when the agent is disabled.
+ * No-op if the config key has no existing variant entry or the variant is already present.
+ */
+export declare function addBuiltinNameVariant(configKey: string, variant: string): void;
 /**
  * Remove lines from a prompt that reference disabled agents.
  * Only strips lines where an agent name appears as a standalone concept

package/dist/config/schema.d.ts

@@ -17,6 +17,7 @@ export declare const AgentOverrideConfigSchema: z.ZodObject<{
         all: "all";
     }>>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
+    display_name: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export declare const AgentOverridesSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
     model: z.ZodOptional<z.ZodString>;
@@ -36,6 +37,7 @@ export declare const AgentOverridesSchema: z.ZodRecord<z.ZodString, z.ZodObject<
         all: "all";
     }>>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
+    display_name: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>>;
 export declare const CategoryConfigSchema: z.ZodObject<{
     description: z.ZodOptional<z.ZodString>;
@@ -82,6 +84,7 @@ export declare const ExperimentalConfigSchema: z.ZodObject<{
     plugin_load_timeout_ms: z.ZodOptional<z.ZodNumber>;
     context_window_warning_threshold: z.ZodOptional<z.ZodNumber>;
     context_window_critical_threshold: z.ZodOptional<z.ZodNumber>;
+    task_system: z.ZodDefault<z.ZodBoolean>;
 }, z.core.$strip>;
 export declare const DelegationTriggerSchema: z.ZodObject<{
     domain: z.ZodString;
@@ -180,6 +183,7 @@ export declare const WeaveConfigSchema: z.ZodObject<{
             all: "all";
         }>>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
+        display_name: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>>;
     custom_agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
         prompt: z.ZodOptional<z.ZodString>;
@@ -255,6 +259,7 @@ export declare const WeaveConfigSchema: z.ZodObject<{
         plugin_load_timeout_ms: z.ZodOptional<z.ZodNumber>;
         context_window_warning_threshold: z.ZodOptional<z.ZodNumber>;
         context_window_critical_threshold: z.ZodOptional<z.ZodNumber>;
+        task_system: z.ZodDefault<z.ZodBoolean>;
     }, z.core.$strip>>;
     workflows: z.ZodOptional<z.ZodObject<{
         disabled_workflows: z.ZodOptional<z.ZodArray<z.ZodString>>;

package/dist/features/evals/baseline.d.ts

@@ -0,0 +1,4 @@
+import type { BaselineComparison, BaselineComparisonOptions, DeterministicBaseline, EvalRunResult } from "./types";
+export declare function deriveDeterministicBaseline(run: EvalRunResult): DeterministicBaseline;
+export declare function readDeterministicBaseline(filePath: string): DeterministicBaseline;
+export declare function compareDeterministicBaseline(baseline: DeterministicBaseline, run: EvalRunResult, options?: BaselineComparisonOptions): BaselineComparison;

package/dist/features/evals/evaluators/deterministic.d.ts

@@ -0,0 +1,2 @@
+import type { AssertionResult, EvalArtifacts, EvaluatorSpec } from "../types";
+export declare function runDeterministicEvaluator(spec: EvaluatorSpec, artifacts: EvalArtifacts): AssertionResult[];

package/dist/features/evals/evaluators/llm-judge.d.ts

@@ -0,0 +1,2 @@
+import type { AssertionResult, EvalArtifacts, LlmJudgeEvaluator } from "../types";
+export declare function runLlmJudgeEvaluator(spec: LlmJudgeEvaluator, artifacts: EvalArtifacts): AssertionResult[];

package/dist/features/evals/executors/model-response.d.ts

@@ -0,0 +1,2 @@
+import type { EvalArtifacts, ExecutionContext, ModelResponseExecutor, ResolvedTarget } from "../types";
+export declare function executeModelResponse(resolvedTarget: ResolvedTarget, executor: ModelResponseExecutor, _context: ExecutionContext): EvalArtifacts;

package/dist/features/evals/executors/prompt-renderer.d.ts

@@ -0,0 +1,2 @@
+import type { EvalArtifacts, ExecutionContext, ExecutorSpec, ResolvedTarget } from "../types";
+export declare function executePromptRender(resolvedTarget: ResolvedTarget, executor: ExecutorSpec, _context: ExecutionContext): EvalArtifacts;

package/dist/features/evals/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Phase 1 eval harness for deterministic prompt-contract coverage.
+ *
+ * Extension points are intentionally registry-based:
+ * - add new target `kind` values in `types.ts` + `schema.ts`
+ * - add new executor handlers in `runner.ts`
+ * - add new evaluator handlers in `evaluators/`
+ * - keep `EvalRunResult` top-level keys stable for future baselines
+ *
+ * Promptfoo, if adopted later, should plug in behind executor/judge adapters.
+ */
+export type { EvalPhase, EvalTarget, ExecutorSpec, EvaluatorSpec, EvalSuiteManifest, EvalCase, LoadedEvalCase, LoadedEvalSuiteManifest, EvalArtifacts, AssertionResult, EvalCaseResult, EvalRunResult, EvalRunSummary, RunEvalSuiteOptions, RunnerFilters, } from "./types";
+export { EvalCaseSchema, EvalSuiteManifestSchema, EvalRunResultSchema } from "./schema";
+export { EvalConfigError, loadEvalSuiteManifest, loadEvalCasesForSuite, resolveSuitePath } from "./loader";
+export { resolveBuiltinAgentTarget } from "./targets/builtin-agent-target";
+export { executePromptRender } from "./executors/prompt-renderer";
+export { executeModelResponse } from "./executors/model-response";
+export { runDeterministicEvaluator } from "./evaluators/deterministic";
+export { runLlmJudgeEvaluator } from "./evaluators/llm-judge";
+export { deriveDeterministicBaseline, readDeterministicBaseline, compareDeterministicBaseline } from "./baseline";
+export { ensureEvalStorageDir, getDefaultEvalRunPath, writeEvalRunResult } from "./storage";
+export { formatEvalSummary } from "./reporter";
+export type { RunEvalSuiteOutput } from "./runner";
+export { runEvalSuite } from "./runner";

package/dist/features/evals/loader.d.ts

@@ -0,0 +1,8 @@
+import type { LoadedEvalCase, LoadedEvalSuiteManifest } from "./types";
+export declare class EvalConfigError extends Error {
+    constructor(message: string);
+}
+export declare function resolveSuitePath(directory: string, suite: string): string;
+export declare function loadEvalSuiteManifest(directory: string, suite: string): LoadedEvalSuiteManifest;
+export declare function loadEvalCaseFile(directory: string, filePath: string): LoadedEvalCase;
+export declare function loadEvalCasesForSuite(directory: string, suite: LoadedEvalSuiteManifest): LoadedEvalCase[];

package/dist/features/evals/reporter.d.ts

@@ -0,0 +1,2 @@
+import type { EvalRunResult } from "./types";
+export declare function formatEvalSummary(result: EvalRunResult): string;

package/dist/features/evals/runner.d.ts

@@ -0,0 +1,7 @@
+import type { EvalRunResult, RunEvalSuiteOptions } from "./types";
+export interface RunEvalSuiteOutput {
+    result: EvalRunResult;
+    artifactPath: string;
+    consoleSummary: string;
+}
+export declare function runEvalSuite(options: RunEvalSuiteOptions): RunEvalSuiteOutput;

package/dist/features/evals/schema.d.ts

@@ -0,0 +1,478 @@
+import { z } from "zod";
+export declare const EvalPhaseSchema: z.ZodEnum<{
+    phase1: "phase1";
+    phase2: "phase2";
+    phase3: "phase3";
+    phase4: "phase4";
+}>;
+export declare const BuiltinAgentPromptVariantSchema: z.ZodObject<{
+    disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const BuiltinAgentPromptTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"builtin-agent-prompt">;
+    agent: z.ZodEnum<{
+        pattern: "pattern";
+        loom: "loom";
+        tapestry: "tapestry";
+        thread: "thread";
+        spindle: "spindle";
+        weft: "weft";
+        warp: "warp";
+    }>;
+    variant: z.ZodOptional<z.ZodObject<{
+        disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const CustomAgentPromptTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"custom-agent-prompt">;
+    agentId: z.ZodString;
+}, z.core.$strip>;
+export declare const SingleTurnAgentTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"single-turn-agent">;
+    agent: z.ZodString;
+    input: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const TrajectoryAgentTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-agent">;
+    agent: z.ZodString;
+    scenarioRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvalTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"builtin-agent-prompt">;
+    agent: z.ZodEnum<{
+        pattern: "pattern";
+        loom: "loom";
+        tapestry: "tapestry";
+        thread: "thread";
+        spindle: "spindle";
+        weft: "weft";
+        warp: "warp";
+    }>;
+    variant: z.ZodOptional<z.ZodObject<{
+        disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"custom-agent-prompt">;
+    agentId: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"single-turn-agent">;
+    agent: z.ZodString;
+    input: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-agent">;
+    agent: z.ZodString;
+    scenarioRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>], "kind">;
+export declare const PromptRenderExecutorSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"prompt-render">;
+}, z.core.$strip>;
+export declare const ModelResponseExecutorSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"model-response">;
+    provider: z.ZodString;
+    model: z.ZodString;
+    input: z.ZodString;
+}, z.core.$strip>;
+export declare const TrajectoryRunExecutorSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-run">;
+    scenarioRef: z.ZodString;
+}, z.core.$strip>;
+export declare const ExecutorSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"prompt-render">;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"model-response">;
+    provider: z.ZodString;
+    model: z.ZodString;
+    input: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-run">;
+    scenarioRef: z.ZodString;
+}, z.core.$strip>], "kind">;
+export declare const ContainsAllEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const ContainsAnyEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-any">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const ExcludesAllEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"excludes-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const SectionContainsAllEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"section-contains-all">;
+    section: z.ZodString;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const OrderedContainsEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"ordered-contains">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const XmlSectionsPresentEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"xml-sections-present">;
+    sections: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const ToolPolicyEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"tool-policy">;
+    expectations: z.ZodRecord<z.ZodString, z.ZodBoolean>;
+}, z.core.$strip>;
+export declare const MinLengthEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"min-length">;
+    min: z.ZodNumber;
+}, z.core.$strip>;
+export declare const LlmJudgeEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"llm-judge">;
+    rubricRef: z.ZodOptional<z.ZodString>;
+    expectedContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    forbiddenContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const BaselineDiffEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"baseline-diff">;
+    baselineRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const TrajectoryAssertionEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"trajectory-assertion">;
+    assertionRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvaluatorSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-any">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"excludes-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"section-contains-all">;
+    section: z.ZodString;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"ordered-contains">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"xml-sections-present">;
+    sections: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"tool-policy">;
+    expectations: z.ZodRecord<z.ZodString, z.ZodBoolean>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"min-length">;
+    min: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"llm-judge">;
+    rubricRef: z.ZodOptional<z.ZodString>;
+    expectedContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    forbiddenContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"baseline-diff">;
+    baselineRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"trajectory-assertion">;
+    assertionRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>], "kind">;
+export declare const EvalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    title: z.ZodString;
+    phase: z.ZodEnum<{
+        phase1: "phase1";
+        phase2: "phase2";
+        phase3: "phase3";
+        phase4: "phase4";
+    }>;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"builtin-agent-prompt">;
+        agent: z.ZodEnum<{
+            pattern: "pattern";
+            loom: "loom";
+            tapestry: "tapestry";
+            thread: "thread";
+            spindle: "spindle";
+            weft: "weft";
+            warp: "warp";
+        }>;
+        variant: z.ZodOptional<z.ZodObject<{
+            disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"custom-agent-prompt">;
+        agentId: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"single-turn-agent">;
+        agent: z.ZodString;
+        input: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"trajectory-agent">;
+        agent: z.ZodString;
+        scenarioRef: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>], "kind">;
+    executor: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"prompt-render">;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"model-response">;
+        provider: z.ZodString;
+        model: z.ZodString;
+        input: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"trajectory-run">;
+        scenarioRef: z.ZodString;
+    }, z.core.$strip>], "kind">;
+    evaluators: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"contains-all">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"contains-any">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"excludes-all">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"section-contains-all">;
+        section: z.ZodString;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"ordered-contains">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"xml-sections-present">;
+        sections: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"tool-policy">;
+        expectations: z.ZodRecord<z.ZodString, z.ZodBoolean>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"min-length">;
+        min: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"llm-judge">;
+        rubricRef: z.ZodOptional<z.ZodString>;
+        expectedContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        forbiddenContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"baseline-diff">;
+        baselineRef: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"trajectory-assertion">;
+        assertionRef: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>], "kind">>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    notes: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvalSuiteManifestSchema: z.ZodObject<{
+    id: z.ZodString;
+    title: z.ZodString;
+    phase: z.ZodEnum<{
+        phase1: "phase1";
+        phase2: "phase2";
+        phase3: "phase3";
+        phase4: "phase4";
+    }>;
+    caseFiles: z.ZodArray<z.ZodString>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const AssertionResultSchema: z.ZodObject<{
+    evaluatorKind: z.ZodEnum<{
+        "contains-all": "contains-all";
+        "contains-any": "contains-any";
+        "excludes-all": "excludes-all";
+        "section-contains-all": "section-contains-all";
+        "ordered-contains": "ordered-contains";
+        "xml-sections-present": "xml-sections-present";
+        "tool-policy": "tool-policy";
+        "min-length": "min-length";
+        "llm-judge": "llm-judge";
+        "baseline-diff": "baseline-diff";
+        "trajectory-assertion": "trajectory-assertion";
+    }>;
+    passed: z.ZodBoolean;
+    score: z.ZodNumber;
+    maxScore: z.ZodNumber;
+    message: z.ZodString;
+}, z.core.$strip>;
+export declare const EvalArtifactsSchema: z.ZodObject<{
+    renderedPrompt: z.ZodOptional<z.ZodString>;
+    agentMetadata: z.ZodOptional<z.ZodObject<{
+        agent: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+        sourceKind: z.ZodEnum<{
+            default: "default";
+            composer: "composer";
+        }>;
+    }, z.core.$strip>>;
+    toolPolicy: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
+    promptLength: z.ZodOptional<z.ZodNumber>;
+    modelOutput: z.ZodOptional<z.ZodString>;
+    judgeOutput: z.ZodOptional<z.ZodString>;
+    trace: z.ZodOptional<z.ZodUnknown>;
+    tokens: z.ZodOptional<z.ZodNumber>;
+    cost: z.ZodOptional<z.ZodNumber>;
+    baselineDelta: z.ZodOptional<z.ZodUnknown>;
+}, z.core.$strip>;
+export declare const EvalCaseResultSchema: z.ZodObject<{
+    caseId: z.ZodString;
+    status: z.ZodEnum<{
+        error: "error";
+        failed: "failed";
+        passed: "passed";
+    }>;
+    score: z.ZodNumber;
+    normalizedScore: z.ZodNumber;
+    maxScore: z.ZodNumber;
+    durationMs: z.ZodNumber;
+    artifacts: z.ZodObject<{
+        renderedPrompt: z.ZodOptional<z.ZodString>;
+        agentMetadata: z.ZodOptional<z.ZodObject<{
+            agent: z.ZodString;
+            description: z.ZodOptional<z.ZodString>;
+            sourceKind: z.ZodEnum<{
+                default: "default";
+                composer: "composer";
+            }>;
+        }, z.core.$strip>>;
+        toolPolicy: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
+        promptLength: z.ZodOptional<z.ZodNumber>;
+        modelOutput: z.ZodOptional<z.ZodString>;
+        judgeOutput: z.ZodOptional<z.ZodString>;
+        trace: z.ZodOptional<z.ZodUnknown>;
+        tokens: z.ZodOptional<z.ZodNumber>;
+        cost: z.ZodOptional<z.ZodNumber>;
+        baselineDelta: z.ZodOptional<z.ZodUnknown>;
+    }, z.core.$strip>;
+    assertionResults: z.ZodArray<z.ZodObject<{
+        evaluatorKind: z.ZodEnum<{
+            "contains-all": "contains-all";
+            "contains-any": "contains-any";
+            "excludes-all": "excludes-all";
+            "section-contains-all": "section-contains-all";
+            "ordered-contains": "ordered-contains";
+            "xml-sections-present": "xml-sections-present";
+            "tool-policy": "tool-policy";
+            "min-length": "min-length";
+            "llm-judge": "llm-judge";
+            "baseline-diff": "baseline-diff";
+            "trajectory-assertion": "trajectory-assertion";
+        }>;
+        passed: z.ZodBoolean;
+        score: z.ZodNumber;
+        maxScore: z.ZodNumber;
+        message: z.ZodString;
+    }, z.core.$strip>>;
+    errors: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvalRunSummarySchema: z.ZodObject<{
+    totalCases: z.ZodNumber;
+    passedCases: z.ZodNumber;
+    failedCases: z.ZodNumber;
+    errorCases: z.ZodNumber;
+    totalScore: z.ZodNumber;
+    normalizedScore: z.ZodNumber;
+    maxScore: z.ZodNumber;
+}, z.core.$strip>;
+export declare const EvalRunResultSchema: z.ZodObject<{
+    runId: z.ZodString;
+    startedAt: z.ZodString;
+    finishedAt: z.ZodString;
+    suiteId: z.ZodString;
+    phase: z.ZodEnum<{
+        phase1: "phase1";
+        phase2: "phase2";
+        phase3: "phase3";
+        phase4: "phase4";
+    }>;
+    summary: z.ZodObject<{
+        totalCases: z.ZodNumber;
+        passedCases: z.ZodNumber;
+        failedCases: z.ZodNumber;
+        errorCases: z.ZodNumber;
+        totalScore: z.ZodNumber;
+        normalizedScore: z.ZodNumber;
+        maxScore: z.ZodNumber;
+    }, z.core.$strip>;
+    caseResults: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        status: z.ZodEnum<{
+            error: "error";
+            failed: "failed";
+            passed: "passed";
+        }>;
+        score: z.ZodNumber;
+        normalizedScore: z.ZodNumber;
+        maxScore: z.ZodNumber;
+        durationMs: z.ZodNumber;
+        artifacts: z.ZodObject<{
+            renderedPrompt: z.ZodOptional<z.ZodString>;
+            agentMetadata: z.ZodOptional<z.ZodObject<{
+                agent: z.ZodString;
+                description: z.ZodOptional<z.ZodString>;
+                sourceKind: z.ZodEnum<{
+                    default: "default";
+                    composer: "composer";
+                }>;
+            }, z.core.$strip>>;
+            toolPolicy: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
+            promptLength: z.ZodOptional<z.ZodNumber>;
+            modelOutput: z.ZodOptional<z.ZodString>;
+            judgeOutput: z.ZodOptional<z.ZodString>;
+            trace: z.ZodOptional<z.ZodUnknown>;
+            tokens: z.ZodOptional<z.ZodNumber>;
+            cost: z.ZodOptional<z.ZodNumber>;
+            baselineDelta: z.ZodOptional<z.ZodUnknown>;
+        }, z.core.$strip>;
+        assertionResults: z.ZodArray<z.ZodObject<{
+            evaluatorKind: z.ZodEnum<{
+                "contains-all": "contains-all";
+                "contains-any": "contains-any";
+                "excludes-all": "excludes-all";
+                "section-contains-all": "section-contains-all";
+                "ordered-contains": "ordered-contains";
+                "xml-sections-present": "xml-sections-present";
+                "tool-policy": "tool-policy";
+                "min-length": "min-length";
+                "llm-judge": "llm-judge";
+                "baseline-diff": "baseline-diff";
+                "trajectory-assertion": "trajectory-assertion";
+            }>;
+            passed: z.ZodBoolean;
+            score: z.ZodNumber;
+            maxScore: z.ZodNumber;
+            message: z.ZodString;
+        }, z.core.$strip>>;
+        errors: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare function formatSchemaIssues(filePath: string, issues: z.ZodIssue[]): string;
+export declare const AllowedEvalTargetKinds: ("builtin-agent-prompt" | "custom-agent-prompt" | "single-turn-agent" | "trajectory-agent")[];
+export declare const AllowedExecutorKinds: ("prompt-render" | "model-response" | "trajectory-run")[];
+export declare const AllowedEvaluatorKinds: ("contains-all" | "contains-any" | "excludes-all" | "section-contains-all" | "ordered-contains" | "xml-sections-present" | "tool-policy" | "min-length" | "llm-judge" | "baseline-diff" | "trajectory-assertion")[];

package/dist/features/evals/storage.d.ts

@@ -0,0 +1,7 @@
+import type { EvalRunResult } from "./types";
+export declare const EVALS_DIR = ".weave/evals";
+export declare const EVAL_RUNS_DIR = ".weave/evals/runs";
+export declare const EVAL_LATEST_FILE = ".weave/evals/latest.json";
+export declare function ensureEvalStorageDir(directory: string): string;
+export declare function getDefaultEvalRunPath(directory: string, runId: string): string;
+export declare function writeEvalRunResult(directory: string, result: EvalRunResult, outputPath?: string): string;

package/dist/features/evals/targets/builtin-agent-target.d.ts

@@ -0,0 +1,2 @@
+import type { BuiltinAgentPromptTarget, ResolvedTarget } from "../types";
+export declare function resolveBuiltinAgentTarget(target: BuiltinAgentPromptTarget): ResolvedTarget;