@opencode_weave/weave 0.7.0 → 0.7.1

package/dist/agents/agent-builder.d.ts

@@ -8,12 +8,34 @@ export type BuildAgentOptions = {
     resolveSkills?: ResolveSkillsFn;
     disabledAgents?: Set<string>;
 };
+/**
+ * Map from agent config key (lowercase) to display name variants that
+ * might appear in prompt text. Used by stripDisabledAgentReferences to
+ * remove lines that mention disabled agents.
+ *
+ * Exported for test cleanup — tests that call addBuiltinNameVariant
+ * must restore original arrays in afterEach to avoid state pollution.
+ */
+export declare const AGENT_NAME_VARIANTS: Record<string, string[]>;
+/**
+ * Reset the mutable name variants map to its initial state.
+ * Used by tests to prevent cross-test state pollution.
+ */
+export declare function resetNameVariants(): void;
 /**
  * Register name variants for a custom agent so that
  * `stripDisabledAgentReferences` can strip its references from prompts.
  * Does not override existing (builtin) entries.
  */
 export declare function registerAgentNameVariants(name: string, variants?: string[]): void;
+/**
+ * Add additional name variants for a builtin agent.
+ * Used when a user sets a custom display_name — the custom name
+ * must be included in variants so stripDisabledAgentReferences
+ * can match it when the agent is disabled.
+ * No-op if the config key has no existing variant entry or the variant is already present.
+ */
+export declare function addBuiltinNameVariant(configKey: string, variant: string): void;
 /**
  * Remove lines from a prompt that reference disabled agents.
  * Only strips lines where an agent name appears as a standalone concept

package/dist/config/schema.d.ts

@@ -17,6 +17,7 @@ export declare const AgentOverrideConfigSchema: z.ZodObject<{
         all: "all";
     }>>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
+    display_name: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export declare const AgentOverridesSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
     model: z.ZodOptional<z.ZodString>;
@@ -36,6 +37,7 @@ export declare const AgentOverridesSchema: z.ZodRecord<z.ZodString, z.ZodObject<
         all: "all";
     }>>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
+    display_name: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>>;
 export declare const CategoryConfigSchema: z.ZodObject<{
     description: z.ZodOptional<z.ZodString>;
@@ -180,6 +182,7 @@ export declare const WeaveConfigSchema: z.ZodObject<{
             all: "all";
         }>>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
+        display_name: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>>;
     custom_agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
         prompt: z.ZodOptional<z.ZodString>;

package/dist/features/evals/baseline.d.ts

@@ -0,0 +1,4 @@
+import type { BaselineComparison, BaselineComparisonOptions, DeterministicBaseline, EvalRunResult } from "./types";
+export declare function deriveDeterministicBaseline(run: EvalRunResult): DeterministicBaseline;
+export declare function readDeterministicBaseline(filePath: string): DeterministicBaseline;
+export declare function compareDeterministicBaseline(baseline: DeterministicBaseline, run: EvalRunResult, options?: BaselineComparisonOptions): BaselineComparison;

package/dist/features/evals/evaluators/deterministic.d.ts

@@ -0,0 +1,2 @@
+import type { AssertionResult, EvalArtifacts, EvaluatorSpec } from "../types";
+export declare function runDeterministicEvaluator(spec: EvaluatorSpec, artifacts: EvalArtifacts): AssertionResult[];

package/dist/features/evals/evaluators/llm-judge.d.ts

@@ -0,0 +1,2 @@
+import type { AssertionResult, EvalArtifacts, LlmJudgeEvaluator } from "../types";
+export declare function runLlmJudgeEvaluator(spec: LlmJudgeEvaluator, artifacts: EvalArtifacts): AssertionResult[];

package/dist/features/evals/executors/model-response.d.ts

@@ -0,0 +1,2 @@
+import type { EvalArtifacts, ExecutionContext, ModelResponseExecutor, ResolvedTarget } from "../types";
+export declare function executeModelResponse(resolvedTarget: ResolvedTarget, executor: ModelResponseExecutor, _context: ExecutionContext): EvalArtifacts;

package/dist/features/evals/executors/prompt-renderer.d.ts

@@ -0,0 +1,2 @@
+import type { EvalArtifacts, ExecutionContext, ExecutorSpec, ResolvedTarget } from "../types";
+export declare function executePromptRender(resolvedTarget: ResolvedTarget, executor: ExecutorSpec, _context: ExecutionContext): EvalArtifacts;

package/dist/features/evals/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Phase 1 eval harness for deterministic prompt-contract coverage.
+ *
+ * Extension points are intentionally registry-based:
+ * - add new target `kind` values in `types.ts` + `schema.ts`
+ * - add new executor handlers in `runner.ts`
+ * - add new evaluator handlers in `evaluators/`
+ * - keep `EvalRunResult` top-level keys stable for future baselines
+ *
+ * Promptfoo, if adopted later, should plug in behind executor/judge adapters.
+ */
+export type { EvalPhase, EvalTarget, ExecutorSpec, EvaluatorSpec, EvalSuiteManifest, EvalCase, LoadedEvalCase, LoadedEvalSuiteManifest, EvalArtifacts, AssertionResult, EvalCaseResult, EvalRunResult, EvalRunSummary, RunEvalSuiteOptions, RunnerFilters, } from "./types";
+export { EvalCaseSchema, EvalSuiteManifestSchema, EvalRunResultSchema } from "./schema";
+export { EvalConfigError, loadEvalSuiteManifest, loadEvalCasesForSuite, resolveSuitePath } from "./loader";
+export { resolveBuiltinAgentTarget } from "./targets/builtin-agent-target";
+export { executePromptRender } from "./executors/prompt-renderer";
+export { executeModelResponse } from "./executors/model-response";
+export { runDeterministicEvaluator } from "./evaluators/deterministic";
+export { runLlmJudgeEvaluator } from "./evaluators/llm-judge";
+export { deriveDeterministicBaseline, readDeterministicBaseline, compareDeterministicBaseline } from "./baseline";
+export { ensureEvalStorageDir, getDefaultEvalRunPath, writeEvalRunResult } from "./storage";
+export { formatEvalSummary } from "./reporter";
+export type { RunEvalSuiteOutput } from "./runner";
+export { runEvalSuite } from "./runner";

package/dist/features/evals/loader.d.ts

@@ -0,0 +1,8 @@
+import type { LoadedEvalCase, LoadedEvalSuiteManifest } from "./types";
+export declare class EvalConfigError extends Error {
+    constructor(message: string);
+}
+export declare function resolveSuitePath(directory: string, suite: string): string;
+export declare function loadEvalSuiteManifest(directory: string, suite: string): LoadedEvalSuiteManifest;
+export declare function loadEvalCaseFile(directory: string, filePath: string): LoadedEvalCase;
+export declare function loadEvalCasesForSuite(directory: string, suite: LoadedEvalSuiteManifest): LoadedEvalCase[];

package/dist/features/evals/reporter.d.ts

@@ -0,0 +1,2 @@
+import type { EvalRunResult } from "./types";
+export declare function formatEvalSummary(result: EvalRunResult): string;

package/dist/features/evals/runner.d.ts

@@ -0,0 +1,7 @@
+import type { EvalRunResult, RunEvalSuiteOptions } from "./types";
+export interface RunEvalSuiteOutput {
+    result: EvalRunResult;
+    artifactPath: string;
+    consoleSummary: string;
+}
+export declare function runEvalSuite(options: RunEvalSuiteOptions): RunEvalSuiteOutput;

package/dist/features/evals/schema.d.ts

@@ -0,0 +1,478 @@
+import { z } from "zod";
+export declare const EvalPhaseSchema: z.ZodEnum<{
+    phase1: "phase1";
+    phase2: "phase2";
+    phase3: "phase3";
+    phase4: "phase4";
+}>;
+export declare const BuiltinAgentPromptVariantSchema: z.ZodObject<{
+    disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const BuiltinAgentPromptTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"builtin-agent-prompt">;
+    agent: z.ZodEnum<{
+        pattern: "pattern";
+        loom: "loom";
+        tapestry: "tapestry";
+        thread: "thread";
+        spindle: "spindle";
+        weft: "weft";
+        warp: "warp";
+    }>;
+    variant: z.ZodOptional<z.ZodObject<{
+        disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const CustomAgentPromptTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"custom-agent-prompt">;
+    agentId: z.ZodString;
+}, z.core.$strip>;
+export declare const SingleTurnAgentTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"single-turn-agent">;
+    agent: z.ZodString;
+    input: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const TrajectoryAgentTargetSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-agent">;
+    agent: z.ZodString;
+    scenarioRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvalTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"builtin-agent-prompt">;
+    agent: z.ZodEnum<{
+        pattern: "pattern";
+        loom: "loom";
+        tapestry: "tapestry";
+        thread: "thread";
+        spindle: "spindle";
+        weft: "weft";
+        warp: "warp";
+    }>;
+    variant: z.ZodOptional<z.ZodObject<{
+        disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"custom-agent-prompt">;
+    agentId: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"single-turn-agent">;
+    agent: z.ZodString;
+    input: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-agent">;
+    agent: z.ZodString;
+    scenarioRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>], "kind">;
+export declare const PromptRenderExecutorSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"prompt-render">;
+}, z.core.$strip>;
+export declare const ModelResponseExecutorSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"model-response">;
+    provider: z.ZodString;
+    model: z.ZodString;
+    input: z.ZodString;
+}, z.core.$strip>;
+export declare const TrajectoryRunExecutorSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-run">;
+    scenarioRef: z.ZodString;
+}, z.core.$strip>;
+export declare const ExecutorSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"prompt-render">;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"model-response">;
+    provider: z.ZodString;
+    model: z.ZodString;
+    input: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"trajectory-run">;
+    scenarioRef: z.ZodString;
+}, z.core.$strip>], "kind">;
+export declare const ContainsAllEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const ContainsAnyEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-any">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const ExcludesAllEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"excludes-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const SectionContainsAllEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"section-contains-all">;
+    section: z.ZodString;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const OrderedContainsEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"ordered-contains">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const XmlSectionsPresentEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"xml-sections-present">;
+    sections: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const ToolPolicyEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"tool-policy">;
+    expectations: z.ZodRecord<z.ZodString, z.ZodBoolean>;
+}, z.core.$strip>;
+export declare const MinLengthEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"min-length">;
+    min: z.ZodNumber;
+}, z.core.$strip>;
+export declare const LlmJudgeEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"llm-judge">;
+    rubricRef: z.ZodOptional<z.ZodString>;
+    expectedContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    forbiddenContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const BaselineDiffEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"baseline-diff">;
+    baselineRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const TrajectoryAssertionEvaluatorSchema: z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"trajectory-assertion">;
+    assertionRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvaluatorSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"contains-any">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"excludes-all">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"section-contains-all">;
+    section: z.ZodString;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"ordered-contains">;
+    patterns: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"xml-sections-present">;
+    sections: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"tool-policy">;
+    expectations: z.ZodRecord<z.ZodString, z.ZodBoolean>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"min-length">;
+    min: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"llm-judge">;
+    rubricRef: z.ZodOptional<z.ZodString>;
+    expectedContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    forbiddenContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"baseline-diff">;
+    baselineRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    weight: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodLiteral<"trajectory-assertion">;
+    assertionRef: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>], "kind">;
+export declare const EvalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    title: z.ZodString;
+    phase: z.ZodEnum<{
+        phase1: "phase1";
+        phase2: "phase2";
+        phase3: "phase3";
+        phase4: "phase4";
+    }>;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"builtin-agent-prompt">;
+        agent: z.ZodEnum<{
+            pattern: "pattern";
+            loom: "loom";
+            tapestry: "tapestry";
+            thread: "thread";
+            spindle: "spindle";
+            weft: "weft";
+            warp: "warp";
+        }>;
+        variant: z.ZodOptional<z.ZodObject<{
+            disabledAgents: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"custom-agent-prompt">;
+        agentId: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"single-turn-agent">;
+        agent: z.ZodString;
+        input: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"trajectory-agent">;
+        agent: z.ZodString;
+        scenarioRef: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>], "kind">;
+    executor: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"prompt-render">;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"model-response">;
+        provider: z.ZodString;
+        model: z.ZodString;
+        input: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"trajectory-run">;
+        scenarioRef: z.ZodString;
+    }, z.core.$strip>], "kind">;
+    evaluators: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"contains-all">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"contains-any">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"excludes-all">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"section-contains-all">;
+        section: z.ZodString;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"ordered-contains">;
+        patterns: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"xml-sections-present">;
+        sections: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"tool-policy">;
+        expectations: z.ZodRecord<z.ZodString, z.ZodBoolean>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"min-length">;
+        min: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"llm-judge">;
+        rubricRef: z.ZodOptional<z.ZodString>;
+        expectedContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        forbiddenContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"baseline-diff">;
+        baselineRef: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        weight: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodLiteral<"trajectory-assertion">;
+        assertionRef: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>], "kind">>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    notes: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvalSuiteManifestSchema: z.ZodObject<{
+    id: z.ZodString;
+    title: z.ZodString;
+    phase: z.ZodEnum<{
+        phase1: "phase1";
+        phase2: "phase2";
+        phase3: "phase3";
+        phase4: "phase4";
+    }>;
+    caseFiles: z.ZodArray<z.ZodString>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const AssertionResultSchema: z.ZodObject<{
+    evaluatorKind: z.ZodEnum<{
+        "contains-all": "contains-all";
+        "contains-any": "contains-any";
+        "excludes-all": "excludes-all";
+        "section-contains-all": "section-contains-all";
+        "ordered-contains": "ordered-contains";
+        "xml-sections-present": "xml-sections-present";
+        "tool-policy": "tool-policy";
+        "min-length": "min-length";
+        "llm-judge": "llm-judge";
+        "baseline-diff": "baseline-diff";
+        "trajectory-assertion": "trajectory-assertion";
+    }>;
+    passed: z.ZodBoolean;
+    score: z.ZodNumber;
+    maxScore: z.ZodNumber;
+    message: z.ZodString;
+}, z.core.$strip>;
+export declare const EvalArtifactsSchema: z.ZodObject<{
+    renderedPrompt: z.ZodOptional<z.ZodString>;
+    agentMetadata: z.ZodOptional<z.ZodObject<{
+        agent: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+        sourceKind: z.ZodEnum<{
+            default: "default";
+            composer: "composer";
+        }>;
+    }, z.core.$strip>>;
+    toolPolicy: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
+    promptLength: z.ZodOptional<z.ZodNumber>;
+    modelOutput: z.ZodOptional<z.ZodString>;
+    judgeOutput: z.ZodOptional<z.ZodString>;
+    trace: z.ZodOptional<z.ZodUnknown>;
+    tokens: z.ZodOptional<z.ZodNumber>;
+    cost: z.ZodOptional<z.ZodNumber>;
+    baselineDelta: z.ZodOptional<z.ZodUnknown>;
+}, z.core.$strip>;
+export declare const EvalCaseResultSchema: z.ZodObject<{
+    caseId: z.ZodString;
+    status: z.ZodEnum<{
+        error: "error";
+        failed: "failed";
+        passed: "passed";
+    }>;
+    score: z.ZodNumber;
+    normalizedScore: z.ZodNumber;
+    maxScore: z.ZodNumber;
+    durationMs: z.ZodNumber;
+    artifacts: z.ZodObject<{
+        renderedPrompt: z.ZodOptional<z.ZodString>;
+        agentMetadata: z.ZodOptional<z.ZodObject<{
+            agent: z.ZodString;
+            description: z.ZodOptional<z.ZodString>;
+            sourceKind: z.ZodEnum<{
+                default: "default";
+                composer: "composer";
+            }>;
+        }, z.core.$strip>>;
+        toolPolicy: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
+        promptLength: z.ZodOptional<z.ZodNumber>;
+        modelOutput: z.ZodOptional<z.ZodString>;
+        judgeOutput: z.ZodOptional<z.ZodString>;
+        trace: z.ZodOptional<z.ZodUnknown>;
+        tokens: z.ZodOptional<z.ZodNumber>;
+        cost: z.ZodOptional<z.ZodNumber>;
+        baselineDelta: z.ZodOptional<z.ZodUnknown>;
+    }, z.core.$strip>;
+    assertionResults: z.ZodArray<z.ZodObject<{
+        evaluatorKind: z.ZodEnum<{
+            "contains-all": "contains-all";
+            "contains-any": "contains-any";
+            "excludes-all": "excludes-all";
+            "section-contains-all": "section-contains-all";
+            "ordered-contains": "ordered-contains";
+            "xml-sections-present": "xml-sections-present";
+            "tool-policy": "tool-policy";
+            "min-length": "min-length";
+            "llm-judge": "llm-judge";
+            "baseline-diff": "baseline-diff";
+            "trajectory-assertion": "trajectory-assertion";
+        }>;
+        passed: z.ZodBoolean;
+        score: z.ZodNumber;
+        maxScore: z.ZodNumber;
+        message: z.ZodString;
+    }, z.core.$strip>>;
+    errors: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export declare const EvalRunSummarySchema: z.ZodObject<{
+    totalCases: z.ZodNumber;
+    passedCases: z.ZodNumber;
+    failedCases: z.ZodNumber;
+    errorCases: z.ZodNumber;
+    totalScore: z.ZodNumber;
+    normalizedScore: z.ZodNumber;
+    maxScore: z.ZodNumber;
+}, z.core.$strip>;
+export declare const EvalRunResultSchema: z.ZodObject<{
+    runId: z.ZodString;
+    startedAt: z.ZodString;
+    finishedAt: z.ZodString;
+    suiteId: z.ZodString;
+    phase: z.ZodEnum<{
+        phase1: "phase1";
+        phase2: "phase2";
+        phase3: "phase3";
+        phase4: "phase4";
+    }>;
+    summary: z.ZodObject<{
+        totalCases: z.ZodNumber;
+        passedCases: z.ZodNumber;
+        failedCases: z.ZodNumber;
+        errorCases: z.ZodNumber;
+        totalScore: z.ZodNumber;
+        normalizedScore: z.ZodNumber;
+        maxScore: z.ZodNumber;
+    }, z.core.$strip>;
+    caseResults: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        status: z.ZodEnum<{
+            error: "error";
+            failed: "failed";
+            passed: "passed";
+        }>;
+        score: z.ZodNumber;
+        normalizedScore: z.ZodNumber;
+        maxScore: z.ZodNumber;
+        durationMs: z.ZodNumber;
+        artifacts: z.ZodObject<{
+            renderedPrompt: z.ZodOptional<z.ZodString>;
+            agentMetadata: z.ZodOptional<z.ZodObject<{
+                agent: z.ZodString;
+                description: z.ZodOptional<z.ZodString>;
+                sourceKind: z.ZodEnum<{
+                    default: "default";
+                    composer: "composer";
+                }>;
+            }, z.core.$strip>>;
+            toolPolicy: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
+            promptLength: z.ZodOptional<z.ZodNumber>;
+            modelOutput: z.ZodOptional<z.ZodString>;
+            judgeOutput: z.ZodOptional<z.ZodString>;
+            trace: z.ZodOptional<z.ZodUnknown>;
+            tokens: z.ZodOptional<z.ZodNumber>;
+            cost: z.ZodOptional<z.ZodNumber>;
+            baselineDelta: z.ZodOptional<z.ZodUnknown>;
+        }, z.core.$strip>;
+        assertionResults: z.ZodArray<z.ZodObject<{
+            evaluatorKind: z.ZodEnum<{
+                "contains-all": "contains-all";
+                "contains-any": "contains-any";
+                "excludes-all": "excludes-all";
+                "section-contains-all": "section-contains-all";
+                "ordered-contains": "ordered-contains";
+                "xml-sections-present": "xml-sections-present";
+                "tool-policy": "tool-policy";
+                "min-length": "min-length";
+                "llm-judge": "llm-judge";
+                "baseline-diff": "baseline-diff";
+                "trajectory-assertion": "trajectory-assertion";
+            }>;
+            passed: z.ZodBoolean;
+            score: z.ZodNumber;
+            maxScore: z.ZodNumber;
+            message: z.ZodString;
+        }, z.core.$strip>>;
+        errors: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare function formatSchemaIssues(filePath: string, issues: z.ZodIssue[]): string;
+export declare const AllowedEvalTargetKinds: ("builtin-agent-prompt" | "custom-agent-prompt" | "single-turn-agent" | "trajectory-agent")[];
+export declare const AllowedExecutorKinds: ("prompt-render" | "model-response" | "trajectory-run")[];
+export declare const AllowedEvaluatorKinds: ("contains-all" | "contains-any" | "excludes-all" | "section-contains-all" | "ordered-contains" | "xml-sections-present" | "tool-policy" | "min-length" | "llm-judge" | "baseline-diff" | "trajectory-assertion")[];

package/dist/features/evals/storage.d.ts

@@ -0,0 +1,7 @@
+import type { EvalRunResult } from "./types";
+export declare const EVALS_DIR = ".weave/evals";
+export declare const EVAL_RUNS_DIR = ".weave/evals/runs";
+export declare const EVAL_LATEST_FILE = ".weave/evals/latest.json";
+export declare function ensureEvalStorageDir(directory: string): string;
+export declare function getDefaultEvalRunPath(directory: string, runId: string): string;
+export declare function writeEvalRunResult(directory: string, result: EvalRunResult, outputPath?: string): string;

package/dist/features/evals/targets/builtin-agent-target.d.ts

@@ -0,0 +1,2 @@
+import type { BuiltinAgentPromptTarget, ResolvedTarget } from "../types";
+export declare function resolveBuiltinAgentTarget(target: BuiltinAgentPromptTarget): ResolvedTarget;

package/dist/features/evals/types.d.ts

@@ -0,0 +1,223 @@
+import type { WeaveAgentName } from "../../agents/types";
+export declare const EVAL_PHASES: readonly ["phase1", "phase2", "phase3", "phase4"];
+export type EvalPhase = (typeof EVAL_PHASES)[number];
+export declare const EVAL_TARGET_KINDS: readonly ["builtin-agent-prompt", "custom-agent-prompt", "single-turn-agent", "trajectory-agent"];
+export type EvalTargetKind = (typeof EVAL_TARGET_KINDS)[number];
+export declare const EXECUTOR_KINDS: readonly ["prompt-render", "model-response", "trajectory-run"];
+export type ExecutorKind = (typeof EXECUTOR_KINDS)[number];
+export declare const EVALUATOR_KINDS: readonly ["contains-all", "contains-any", "excludes-all", "section-contains-all", "ordered-contains", "xml-sections-present", "tool-policy", "min-length", "llm-judge", "baseline-diff", "trajectory-assertion"];
+export type EvaluatorKind = (typeof EVALUATOR_KINDS)[number];
+export type BuiltinEvalAgentName = Exclude<WeaveAgentName, "shuttle">;
+export interface BuiltinAgentPromptVariant {
+    disabledAgents?: string[];
+}
+export interface BuiltinAgentPromptTarget {
+    kind: "builtin-agent-prompt";
+    agent: BuiltinEvalAgentName;
+    variant?: BuiltinAgentPromptVariant;
+}
+export interface CustomAgentPromptTarget {
+    kind: "custom-agent-prompt";
+    agentId: string;
+}
+export interface SingleTurnAgentTarget {
+    kind: "single-turn-agent";
+    agent: string;
+    input?: string;
+}
+export interface TrajectoryAgentTarget {
+    kind: "trajectory-agent";
+    agent: string;
+    scenarioRef?: string;
+}
+export type EvalTarget = BuiltinAgentPromptTarget | CustomAgentPromptTarget | SingleTurnAgentTarget | TrajectoryAgentTarget;
+export interface PromptRenderExecutor {
+    kind: "prompt-render";
+}
+export interface ModelResponseExecutor {
+    kind: "model-response";
+    provider: string;
+    model: string;
+    input: string;
+}
+export interface TrajectoryRunExecutor {
+    kind: "trajectory-run";
+    scenarioRef: string;
+}
+export type ExecutorSpec = PromptRenderExecutor | ModelResponseExecutor | TrajectoryRunExecutor;
+export interface WeightedEvaluatorSpec {
+    weight?: number;
+}
+export interface ContainsAllEvaluator extends WeightedEvaluatorSpec {
+    kind: "contains-all";
+    patterns: string[];
+}
+export interface ContainsAnyEvaluator extends WeightedEvaluatorSpec {
+    kind: "contains-any";
+    patterns: string[];
+}
+export interface ExcludesAllEvaluator extends WeightedEvaluatorSpec {
+    kind: "excludes-all";
+    patterns: string[];
+}
+export interface SectionContainsAllEvaluator extends WeightedEvaluatorSpec {
+    kind: "section-contains-all";
+    section: string;
+    patterns: string[];
+}
+export interface OrderedContainsEvaluator extends WeightedEvaluatorSpec {
+    kind: "ordered-contains";
+    patterns: string[];
+}
+export interface XmlSectionsPresentEvaluator extends WeightedEvaluatorSpec {
+    kind: "xml-sections-present";
+    sections: string[];
+}
+export interface ToolPolicyEvaluator extends WeightedEvaluatorSpec {
+    kind: "tool-policy";
+    expectations: Record<string, boolean>;
+}
+export interface MinLengthEvaluator extends WeightedEvaluatorSpec {
+    kind: "min-length";
+    min: number;
+}
+export interface LlmJudgeEvaluator extends WeightedEvaluatorSpec {
+    kind: "llm-judge";
+    rubricRef?: string;
+    expectedContains?: string[];
+    forbiddenContains?: string[];
+}
+export interface BaselineDiffEvaluator extends WeightedEvaluatorSpec {
+    kind: "baseline-diff";
+    baselineRef?: string;
+}
+export interface TrajectoryAssertionEvaluator extends WeightedEvaluatorSpec {
+    kind: "trajectory-assertion";
+    assertionRef?: string;
+}
+export type EvaluatorSpec = ContainsAllEvaluator | ContainsAnyEvaluator | ExcludesAllEvaluator | SectionContainsAllEvaluator | OrderedContainsEvaluator | XmlSectionsPresentEvaluator | ToolPolicyEvaluator | MinLengthEvaluator | LlmJudgeEvaluator | BaselineDiffEvaluator | TrajectoryAssertionEvaluator;
+export interface EvalSuiteManifest {
+    id: string;
+    title: string;
+    phase: EvalPhase;
+    caseFiles: string[];
+    tags?: string[];
+}
+export interface EvalCase {
+    id: string;
+    title: string;
+    phase: EvalPhase;
+    target: EvalTarget;
+    executor: ExecutorSpec;
+    evaluators: EvaluatorSpec[];
+    tags?: string[];
+    notes?: string;
+}
+export interface LoadedEvalSuiteManifest extends EvalSuiteManifest {
+    filePath: string;
+}
+export interface LoadedEvalCase extends EvalCase {
+    filePath: string;
+}
+export interface AgentPromptMetadataArtifact {
+    agent: string;
+    description?: string;
+    sourceKind: "composer" | "default";
+}
+export interface EvalArtifacts {
+    renderedPrompt?: string;
+    agentMetadata?: AgentPromptMetadataArtifact;
+    toolPolicy?: Record<string, boolean>;
+    promptLength?: number;
+    modelOutput?: string;
+    judgeOutput?: string;
+    trace?: unknown;
+    tokens?: number;
+    cost?: number;
+    baselineDelta?: unknown;
+}
+export interface AssertionResult {
+    evaluatorKind: EvaluatorKind;
+    passed: boolean;
+    score: number;
+    maxScore: number;
+    message: string;
+}
+export interface EvalCaseResult {
+    caseId: string;
+    status: "passed" | "failed" | "error";
+    score: number;
+    normalizedScore: number;
+    maxScore: number;
+    durationMs: number;
+    artifacts: EvalArtifacts;
+    assertionResults: AssertionResult[];
+    errors: string[];
+}
+export interface EvalRunSummary {
+    totalCases: number;
+    passedCases: number;
+    failedCases: number;
+    errorCases: number;
+    totalScore: number;
+    normalizedScore: number;
+    maxScore: number;
+}
+export interface EvalRunResult {
+    runId: string;
+    startedAt: string;
+    finishedAt: string;
+    suiteId: string;
+    phase: EvalPhase;
+    summary: EvalRunSummary;
+    caseResults: EvalCaseResult[];
+}
+export interface ResolvedTarget {
+    target: EvalTarget;
+    artifacts: EvalArtifacts;
+}
+export interface ExecutionContext {
+    mode: "local" | "ci" | "hosted";
+    directory: string;
+    outputPath?: string;
+}
+export interface RunnerFilters {
+    caseIds?: string[];
+    agents?: string[];
+    tags?: string[];
+}
+export interface RunEvalSuiteOptions {
+    directory: string;
+    suite: string;
+    filters?: RunnerFilters;
+    outputPath?: string;
+    mode?: ExecutionContext["mode"];
+}
+export interface EvalLoadErrorContext {
+    filePath: string;
+    detail: string;
+}
+export interface DeterministicBaselineCase {
+    caseId: string;
+    status: EvalCaseResult["status"];
+    normalizedScore: number;
+    assertionPassed: number;
+    assertionFailed: number;
+    errorCount: number;
+}
+export interface DeterministicBaseline {
+    version: 1;
+    suiteId: string;
+    phase: EvalPhase;
+    generatedAt: string;
+    normalizedScore: number;
+    cases: DeterministicBaselineCase[];
+}
+export interface BaselineComparisonOptions {
+    scoreDropTolerance?: number;
+}
+export interface BaselineComparison {
+    outcome: "no-regression" | "informational-diff" | "regression";
+    regressions: string[];
+    informational: string[];
+}

package/dist/index.js

@@ -22,7 +22,8 @@ var AgentOverrideConfigSchema = z.object({
   tools: z.record(z.string(), z.boolean()).optional(),
   disable: z.boolean().optional(),
   mode: z.enum(["subagent", "primary", "all"]).optional(),
-  maxTokens: z.number().optional()
+  maxTokens: z.number().optional(),
+  display_name: z.string().optional()
 });
 var AgentOverridesSchema = z.record(z.string(), AgentOverrideConfigSchema);
 var CategoryConfigSchema = z.object({
@@ -223,6 +224,7 @@ var AGENT_DISPLAY_NAMES = {
   weft: "weft"
 };
 var BUILTIN_CONFIG_KEYS = new Set(Object.keys(AGENT_DISPLAY_NAMES));
+var INITIAL_BUILTIN_DISPLAY_NAMES = new Map(Object.entries(AGENT_DISPLAY_NAMES));
 var reverseDisplayNames = null;
 function getReverseDisplayNames() {
   if (reverseDisplayNames === null) {
@@ -234,6 +236,11 @@ function registerAgentDisplayName(configKey, displayName) {
   if (BUILTIN_CONFIG_KEYS.has(configKey)) {
     throw new Error(`Cannot register display name for "${configKey}": it is a built-in agent name`);
   }
+  for (const [builtinKey, initialDisplayName] of INITIAL_BUILTIN_DISPLAY_NAMES) {
+    if (initialDisplayName.toLowerCase() === displayName.toLowerCase()) {
+      throw new Error(`Display name "${displayName}" is reserved for built-in agent "${builtinKey}"`);
+    }
+  }
   const reverse = getReverseDisplayNames();
   const existingKey = reverse[displayName.toLowerCase()];
   if (existingKey !== undefined && BUILTIN_CONFIG_KEYS.has(existingKey)) {
@@ -242,6 +249,13 @@ function registerAgentDisplayName(configKey, displayName) {
   AGENT_DISPLAY_NAMES[configKey] = displayName;
   reverseDisplayNames = null;
 }
+function updateBuiltinDisplayName(configKey, displayName) {
+  if (!BUILTIN_CONFIG_KEYS.has(configKey)) {
+    throw new Error(`Cannot update builtin display name for "${configKey}": not a built-in agent`);
+  }
+  AGENT_DISPLAY_NAMES[configKey] = displayName;
+  reverseDisplayNames = null;
+}
 function getAgentDisplayName(configKey) {
   const exactMatch = AGENT_DISPLAY_NAMES[configKey];
   if (exactMatch !== undefined)
@@ -1725,12 +1739,19 @@ var AGENT_NAME_VARIANTS = {
   loom: ["loom", "Loom"],
   tapestry: ["tapestry", "Tapestry"]
 };
+var INITIAL_NAME_VARIANTS = new Map(Object.entries(AGENT_NAME_VARIANTS).map(([k, v]) => [k, [...v]]));
 function registerAgentNameVariants(name, variants) {
   if (AGENT_NAME_VARIANTS[name])
     return;
   const titleCase = name.charAt(0).toUpperCase() + name.slice(1);
   AGENT_NAME_VARIANTS[name] = variants ?? [name, titleCase];
 }
+function addBuiltinNameVariant(configKey, variant) {
+  const existing = AGENT_NAME_VARIANTS[configKey];
+  if (existing && !existing.includes(variant)) {
+    existing.push(variant);
+  }
+}
 function stripDisabledAgentReferences(prompt, disabled) {
   if (disabled.size === 0)
     return prompt;
@@ -1743,7 +1764,7 @@ function stripDisabledAgentReferences(prompt, disabled) {
   }
   if (disabledVariants.length === 0)
     return prompt;
-  const pattern = new RegExp(`\\b(${disabledVariants.map((v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})\\b`);
+  const pattern = new RegExp(`(?<!\\w)(${disabledVariants.map((v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})(?!\\w)`);
   const lines = prompt.split(`
 `);
   const filtered = lines.filter((line) => !pattern.test(line));
@@ -1988,6 +2009,26 @@ function createManagers(options) {
     fingerprint,
     customAgentMetadata
   });
+  if (pluginConfig.agents) {
+    for (const [name, override] of Object.entries(pluginConfig.agents)) {
+      const displayName = override.display_name?.trim();
+      if (displayName) {
+        try {
+          updateBuiltinDisplayName(name, displayName);
+          addBuiltinNameVariant(name, displayName);
+          if (agents[name]) {
+            agents[name] = { ...agents[name], description: displayName };
+          }
+        } catch (err) {
+          if (err instanceof Error && err.message.includes("not a built-in agent")) {
+            log(`Skipping display_name override for non-builtin agent "${name}"`);
+          } else {
+            throw err;
+          }
+        }
+      }
+    }
+  }
   if (pluginConfig.custom_agents) {
     const disabledSet = new Set(pluginConfig.disabled_agents ?? []);
     for (const [name, customConfig] of Object.entries(pluginConfig.custom_agents)) {

package/dist/shared/agent-display-names.d.ts

@@ -10,6 +10,11 @@
  * registerAgentDisplayName().
  */
 export declare const AGENT_DISPLAY_NAMES: Record<string, string>;
+/**
+ * Reset the mutable display name map to its initial state.
+ * Used by tests to prevent cross-test state pollution.
+ */
+export declare function resetDisplayNames(): void;
 /**
  * Register a display name for an agent config key.
  * Custom agents call this so getAgentDisplayName/getAgentConfigKey work for them.
@@ -18,6 +23,15 @@ export declare const AGENT_DISPLAY_NAMES: Record<string, string>;
  * or if the config key is a built-in agent name.
  */
 export declare function registerAgentDisplayName(configKey: string, displayName: string): void;
+/**
+ * Override the display name for a built-in agent.
+ * Unlike registerAgentDisplayName (which guards against builtin config keys),
+ * this function is specifically for user-configured builtin display names.
+ *
+ * Only accepts known builtin config keys. Throws for unknown keys.
+ * Invalidates the reverse lookup cache so getAgentConfigKey reflects the new name.
+ */
+export declare function updateBuiltinDisplayName(configKey: string, displayName: string): void;
 /**
  * Get display name for an agent config key.
  * Uses case-insensitive lookup for flexibility.

package/dist/shared/index.d.ts

@@ -2,4 +2,4 @@ export type { DeepPartial, Brand } from "./types";
 export { getWeaveVersion } from "./version";
 export { log, getLogFilePath, logDelegation } from "./log";
 export type { DelegationEvent } from "./log";
-export { AGENT_DISPLAY_NAMES, getAgentDisplayName, getAgentConfigKey, registerAgentDisplayName, } from "./agent-display-names";
+export { AGENT_DISPLAY_NAMES, getAgentDisplayName, getAgentConfigKey, registerAgentDisplayName, updateBuiltinDisplayName, resetDisplayNames, } from "./agent-display-names";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencode_weave/weave",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Weave — lean OpenCode plugin with multi-agent orchestration",
   "author": "Weave",
   "license": "MIT",
@@ -25,7 +25,10 @@
     "build": "bun run script/build.ts && tsc --emitDeclarationOnly",
     "clean": "rm -rf dist",
     "typecheck": "tsc --noEmit",
-    "test": "bun test"
+    "test": "bun test",
+    "eval": "bun run script/eval.ts",
+    "eval:smoke": "bun run script/eval.ts --suite pr-smoke",
+    "eval:coverage": "bun run script/verify-eval-coverage.ts"
   },
   "keywords": [
     "opencode",