agentv 5.2.0-next.1 → 5.3.0-next.1

package/dist/{chunk-ALVZQUZP.js → chunk-V52ATPTT.js}

@@ -16,7 +16,7 @@ import {
   prepareSharedWorkspaceSetup,
   readTranscriptJsonl,
   releaseSharedWorkspaceSetup
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-6ZZCDZPD.js";
 
 // ../../packages/core/dist/index.js
 import { readFileSync } from "node:fs";
@@ -6089,4 +6089,4 @@ export {
   parseCopilotEvents,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-ALVZQUZP.js.map
+//# sourceMappingURL=chunk-V52ATPTT.js.map

package/dist/cli.js

@@ -2,13 +2,13 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   runCli
-} from "./chunk-3X357HS4.js";
-import "./chunk-T4AD3H3Y.js";
-import "./chunk-46K2OET3.js";
-import "./chunk-ALVZQUZP.js";
+} from "./chunk-T32NL3E6.js";
+import "./chunk-FVR4RQFK.js";
+import "./chunk-BHKQHG26.js";
+import "./chunk-V52ATPTT.js";
 import {
   killAllTrackedChildren
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-6ZZCDZPD.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";

package/dist/{dist-DEPJOMCA.js → dist-6Z7U473R.js}

@@ -54,7 +54,7 @@ import {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-ALVZQUZP.js";
+} from "./chunk-V52ATPTT.js";
 import {
   AGENTV_CONFIG_FILE_NAME,
   AGENTV_CONFIG_YML_FILE_NAME,
@@ -83,6 +83,7 @@ import {
   ExternalTraceMetadataWireSchema,
   FieldAccuracyGrader,
   GraderRegistry,
+  LEGACY_METRICS_SCHEMA_VERSION,
   LatencyGrader,
   LlmGrader,
   METRICS_JSON_MEDIA_TYPE,
@@ -106,6 +107,7 @@ import {
   ResultRowSchemaError,
   ScriptGrader,
   SkillTriggerGrader,
+  SkillUsedGrader,
   TEST_MESSAGE_ROLES,
   TRACE_EVENT_TYPES,
   TRACE_REDACTION_LEVELS,
@@ -145,6 +147,7 @@ import {
   TraceSourceWireSchema,
   TraceSummaryBoundarySchema,
   TraceToolWireSchema,
+  TrajectoryGrader,
   WorkspaceCreationError,
   addProject,
   aggregateRunDir,
@@ -282,6 +285,7 @@ import {
   normalizeExperimentConfig,
   normalizeExperimentRunOverride,
   normalizeLineEndings,
+  normalizeMetricsArtifactWire,
   normalizeRepoIdentity,
   normalizeResultRow,
   omitExternalTraceMetadataKeys,
@@ -373,7 +377,7 @@ import {
   writeArtifactsFromResults,
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-6ZZCDZPD.js";
 import "./chunk-7BGERE6L.js";
 import {
   DockerWorkspaceProvider
@@ -409,6 +413,7 @@ export {
   ExternalTraceMetadataWireSchema,
   FieldAccuracyGrader,
   GraderRegistry,
+  LEGACY_METRICS_SCHEMA_VERSION,
   LatencyGrader,
   LlmGrader,
   METRICS_JSON_MEDIA_TYPE,
@@ -433,6 +438,7 @@ export {
   RunBudgetTracker,
   ScriptGrader,
   SkillTriggerGrader,
+  SkillUsedGrader,
   TEST_MESSAGE_ROLES,
   TRACE_EVENT_TYPES,
   TRACE_REDACTION_LEVELS,
@@ -473,6 +479,7 @@ export {
   TraceSourceWireSchema,
   TraceSummaryBoundarySchema,
   TraceToolWireSchema,
+  TrajectoryGrader,
   TranscriptProvider,
   WorkspaceCreationError,
   addProject,
@@ -636,6 +643,7 @@ export {
   normalizeExperimentConfig,
   normalizeExperimentRunOverride,
   normalizeLineEndings,
+  normalizeMetricsArtifactWire,
   normalizeRepoIdentity,
   normalizeResultRow,
   normalizeResultsConfig,
@@ -754,4 +762,4 @@ export {
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=dist-DEPJOMCA.js.map
+//# sourceMappingURL=dist-6Z7U473R.js.map

package/dist/index.js

@@ -6,11 +6,11 @@ import {
   runCli,
   shouldRunBeforeSessionHook,
   usesDeprecatedStudioAlias
-} from "./chunk-3X357HS4.js";
-import "./chunk-T4AD3H3Y.js";
-import "./chunk-46K2OET3.js";
-import "./chunk-ALVZQUZP.js";
-import "./chunk-3S6A2RKR.js";
+} from "./chunk-T32NL3E6.js";
+import "./chunk-FVR4RQFK.js";
+import "./chunk-BHKQHG26.js";
+import "./chunk-V52ATPTT.js";
+import "./chunk-6ZZCDZPD.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";

package/dist/{interactive-36ZNMQB2.js → interactive-RUY3OCBI.js}

@@ -8,14 +8,14 @@ import {
   getCategories,
   resolveExistingRunPrimaryPath,
   runEvalCommand
-} from "./chunk-T4AD3H3Y.js";
-import "./chunk-46K2OET3.js";
-import "./chunk-ALVZQUZP.js";
+} from "./chunk-FVR4RQFK.js";
+import "./chunk-BHKQHG26.js";
+import "./chunk-V52ATPTT.js";
 import {
   getAgentvConfigDir,
   listTargetNames,
   readTargetDefinitions
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-6ZZCDZPD.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
@@ -344,4 +344,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
 export {
   launchInteractiveWizard
 };
-//# sourceMappingURL=interactive-36ZNMQB2.js.map
+//# sourceMappingURL=interactive-RUY3OCBI.js.map

package/dist/skills/agentv-bench/SKILL.md

@@ -117,7 +117,7 @@ Start with 2-3 realistic test cases — the kind of thing a real user would actu
 
 Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment.
 
-**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `assert-set`, `script`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
+**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `assert-set`, `skill-used`, `not-skill-used`, `trajectory:*`, `script`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
 
 Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-rubric`.
 

package/dist/skills/agentv-bench/agents/analyzer.md

@@ -123,7 +123,7 @@ If a section has no findings, include the header with "None found." underneath.
 - **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config.
 - **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on.
 - **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-rubric` → deterministic saves the most).
-- **Handle all grader types:** Process `script`, `tool-trajectory`, `llm-rubric`, `rubrics`, `assert-set`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
+- **Handle all grader types:** Process `script`, `trajectory:*`, `skill-used`, `not-skill-used`, `llm-rubric`, `rubrics`, `assert-set`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
 - **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific.
 - **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning.
 

package/dist/skills/agentv-bench/agents/comparator.md

@@ -24,7 +24,7 @@ You are the Blind Comparator for AgentV's evaluation workflow. Your job is to co
 You will receive:
 - `outputs`: Array of evaluation outputs to compare. Each contains:
   - `target_id`: The provider/configuration identifier (DO NOT read this during scoring)
-  - `evaluator_results`: Array of grader scores and details (script, tool-trajectory, llm-rubric, deterministic)
+  - `evaluator_results`: Array of grader scores and details (script, trajectory, skill-use, llm-rubric, deterministic)
   - `workspace_changes`: File changes made during workspace evaluation (if applicable)
   - `tool_calls`: Tool invocations and results from multi-turn conversations (if applicable)
   - `conversation`: Full multi-turn conversation history (if applicable)
@@ -91,7 +91,7 @@ For each content criterion, define:
 **Grader-Specific Scoring** — when grader results are present:
 
 - **script-grader**: Factor in pass/fail results, test coverage, assertion hit rates
-- **tool-trajectory**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
+- **trajectory assertions**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
 - **llm-rubric**: Factor in existing LLM grader scores as a reference signal (not as ground truth)
 - **deterministic**: Factor in exact match / keyword hit rates
 
@@ -235,7 +235,7 @@ Also produce a human-readable markdown summary:
 - **Be evidence-based**: Every score must cite specific evidence from the output.
 - **Evaluate substance over style**: Correct, complete answers with rough formatting score higher than polished but incorrect answers.
 - **Handle missing data gracefully**: If an output lacks workspace changes or tool calls but others have them, score what is present — do not penalize for data the target wasn't expected to produce.
-- **Respect grader signals**: When script-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily.
+- **Respect grader signals**: When script-grader or trajectory assertion results exist, they represent objective ground truth. Weight these heavily.
 
 ## Edge Cases
 

package/dist/skills/agentv-bench/agents/grader.md

@@ -80,8 +80,10 @@ For LLM-graded types: be rigorous and fair. Score based on substance, not exact
 
 | Type | How to evaluate |
 |------|----------------|
-| `tool-trajectory` | Inspect transcript for tool calls, match against expected sequence/mode |
-| `skill-trigger` | Check if the named skill was invoked in tool calls |
+| `trajectory:tool-used` | Inspect transcript for tool presence and minimum call counts |
+| `trajectory:tool-sequence` | Inspect transcript for expected tool call order |
+| `trajectory:tool-args-match` | Inspect transcript for expected tool arguments |
+| `skill-used` / `not-skill-used` | Check whether the named skill was invoked in tool calls |
 
 If transcript data is not available for tool inspection assertions, record `score: null` with a note that transcript data was not captured. Exclude from the weighted average.
 

package/dist/skills/agentv-bench/references/description-optimization.md

@@ -6,7 +6,7 @@ core workflow step.
 
 **Provider compatibility**: Description optimization applies to any agent platform with
 skill-discovery mechanisms — Claude Code, Codex (`.agents/` or `.codex/` folders), Copilot,
-and others. The `skill-trigger` grader checks whether the agent invoked the right skill,
+and others. The `skill-used` and `not-skill-used` graders check whether the agent invoked the right skill,
 regardless of how discovery works on that platform.
 
 ## Step 1: Generate Trigger EVAL.yaml
@@ -36,14 +36,13 @@ tests:
   - id: should-trigger-casual-optimize
     input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it"
     assert:
-      - type: skill-trigger
-        skill: agentv-bench
+      - type: skill-used
+        value: agentv-bench
   - id: should-not-trigger-build-error
     input: "my TypeScript build is failing with type errors in src/auth.ts"
     assert:
-      - type: skill-trigger
-        skill: agentv-bench
-        should_trigger: false
+      - type: not-skill-used
+        value: agentv-bench
 ```
 
 ## Step 2: Review with User

package/dist/skills/agentv-bench/references/environment-adaptation.md

@@ -28,7 +28,7 @@ to any platform with skill-discovery mechanisms. All listed providers support sk
 
 ## Unsupported Providers: Use a Code-Grader
 
-The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out
+The built-in `skill-used` and `not-skill-used` graders cover Claude, Copilot, Pi, Codex and VS Code out
 of the box. For providers with different tool-call formats, write a script-grader that inspects
 the agent's transcript messages or tool call trace.
 
@@ -37,7 +37,7 @@ transcript `messages`, and structured `trace`. Inspect `messages` or `trace.even
 tool calls; reserve `output` for final-answer text checks.
 
 ```yaml
-# Example: script-grader for Codex skill-trigger detection
+# Example: script-grader for Codex skill-use detection
 tests:
   - id: should-trigger-codex
     input: "Analyze this CSV file"

package/dist/skills/agentv-bench/references/eval-yaml-spec.md

@@ -171,18 +171,29 @@ Same as contains variants but explicitly case-insensitive.
 
 ### Tool inspection assertions
 
-#### `tool-trajectory`
+#### `trajectory:tool-used`
 
-- **Fields:** `expected` (array of expected tool calls), `mode` (string: `any_order` | `in_order` | `exact` | `subset` | `superset`)
-- **Recipe:** Inspect AgentV-normalized transcript/tool-call data. Match against expected based on mode.
-- **PASS:** tool calls match expected pattern per mode.
-- **Boundary:** This is an AgentV extension. Promptfoo `trajectory:*`, `tool-call-f1`, `skill-used`, and `trace-*` assertion names are rejected until implemented directly.
+- **Fields:** `value.name` (string, required), `value.min` (number, optional)
+- **Recipe:** Inspect AgentV-normalized transcript/tool-call data and count matching tool calls.
+- **PASS:** matching calls meet the requested minimum.
 
-#### `skill-trigger`
+#### `trajectory:tool-sequence`
 
-- **Fields:** `skill_name` (string, required)
-- **Recipe:** Check if the agent invoked the named skill in its tool calls.
-- **PASS:** skill was triggered.
+- **Fields:** `value.steps` (array of tool names, required), `value.mode` (`in_order` | `exact`)
+- **Recipe:** Inspect AgentV-normalized transcript/tool-call data and compare the observed sequence.
+- **PASS:** tool calls match the expected sequence for the selected mode.
+
+#### `trajectory:tool-args-match`
+
+- **Fields:** `value.name` (string, required), `value.args` (object, required), `value.mode` (`partial` | `exact`)
+- **Recipe:** Inspect matching tool calls and compare arguments.
+- **PASS:** at least one matching call has the expected arguments.
+
+#### `skill-used` / `not-skill-used`
+
+- **Fields:** `value` (skill name string or pattern object, required)
+- **Recipe:** Check whether the agent invoked the named skill in its tool calls.
+- **PASS:** `skill-used` finds the skill; `not-skill-used` confirms it is absent.
 
 ### LLM-judged assertions (require Claude reasoning)
 

package/dist/skills/agentv-eval-migrations/references/breaking-changes.md

@@ -51,9 +51,12 @@ For a v4.42.4-era eval:
 15. Replace authored `preprocessors` and deprecated Promptfoo `postprocess`
     with `transform` at `default_test.options`, `tests[].options`, or the
     assertion that needs the shaped output.
-16. Keep raw cases under `tests` / `tests: file://...`; run full eval suites
+16. Replace `type: skill-trigger` with `skill-used` or `not-skill-used`.
+17. Replace `type: tool-trajectory` with Promptfoo-compatible `trajectory:*`
+    assertions where there is a direct mapping.
+18. Keep raw cases under `tests` / `tests: file://...`; run full eval suites
     directly with CLI multi-file selection and tags.
-17. Validate with `bun apps/cli/src/cli.ts validate <eval-file>`.
+19. Validate with `bun apps/cli/src/cli.ts validate <eval-file>`.
 
 ## Assertions Renamed To `assert`
 
@@ -211,6 +214,101 @@ templates and custom graders can consume it. The breaking authoring change is
 that workers should not depend on missing `assert` plus `criteria` to define
 the whole grading contract for new migrated YAML.
 
+## `skill-trigger` And `tool-trajectory` Replaced By Promptfoo Assertions
+
+### v4.42.4 Shape
+
+Older AgentV YAML used AgentV-specific skill and tool trajectory assertions:
+
+```yaml
+assert:
+  - type: skill-trigger
+    skill: csv-analyzer
+    should_trigger: true
+  - type: tool-trajectory
+    mode: any_order
+    minimums:
+      search: 2
+  - type: tool-trajectory
+    mode: exact
+    expected:
+      - tool: search
+        args:
+          q: agentv
+      - tool: fetch
+```
+
+### Current Shape
+
+Current authored eval YAML uses Promptfoo-compatible assertions:
+
+```yaml
+assert:
+  - type: skill-used
+    value: csv-analyzer
+  - type: trajectory:tool-used
+    value:
+      name: search
+      min: 2
+  - type: trajectory:tool-sequence
+    value:
+      mode: exact
+      steps: [search, fetch]
+  - type: trajectory:tool-args-match
+    value:
+      name: search
+      args:
+        q: agentv
+      mode: partial
+```
+
+### Migration Steps
+
+- Replace `type: skill-trigger`, `skill: X`, `should_trigger: true` with
+  `type: skill-used`, `value: X`.
+- Replace `type: skill-trigger`, `skill: X`, `should_trigger: false` with
+  `type: not-skill-used`, `value: X`.
+- Replace `type: tool-trajectory`, `mode: any_order`, and `minimums: { Tool: N }`
+  with one `trajectory:tool-used` assertion per tool:
+
+  ```yaml
+  - type: trajectory:tool-used
+    value:
+      name: Tool
+      min: N
+  ```
+
+- Replace `mode: in_order` or `mode: exact` plus `expected` tool steps with
+  `trajectory:tool-sequence`:
+
+  ```yaml
+  - type: trajectory:tool-sequence
+    value:
+      mode: exact
+      steps: [search, fetch]
+  ```
+
+- Move old `expected[].args` checks to `trajectory:tool-args-match`. Use
+  `mode: exact` only for exact argument equality; otherwise use Promptfoo's
+  partial argument matching.
+- Do not carry old `max_duration_ms` per-tool latency checks under
+  `trajectory:*`. There is no Promptfoo-compatible equivalent in AgentV yet;
+  use a `script` assertion or track the behavior as future scope.
+
+### Verification
+
+```bash
+bun apps/cli/src/cli.ts validate path/to/eval.eval.yaml
+rg -n "skill-trigger|tool-trajectory" path/to/evals path/to/.agentv/templates
+```
+
+### Compatibility Notes
+
+The runtime still has private compatibility classes and tests for older
+helpers. That does not make `skill-trigger` or `tool-trajectory` valid authored
+eval YAML. Normal Promptfoo-aligned eval files fail validation and parsing with
+the replacement guidance above.
+
 ## Authored `input` Moved To `prompts` Plus Vars
 
 ### v4.42.4 Shape

package/dist/skills/agentv-eval-writer/SKILL.md

@@ -569,22 +569,33 @@ Variables: `{{criteria}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{
 ```
 Use `assert-set` for Promptfoo-aligned assertion grouping. Without `threshold`, the set passes only when every nonzero-weight child assertion passes. With `threshold`, the weighted aggregate score determines the set verdict. Parent `config` is inherited by children, and child `config` keys override parent keys. Do not use `type: composite`; AgentV rejects it.
 
-### tool-trajectory
-```yaml
-- name: tool_check
-  type: tool-trajectory
-  mode: any_order            # any_order | in_order | exact
-  minimums:                  # for any_order
-    knowledgeSearch: 2
-  expected:                  # for in_order/exact
-    - tool: knowledgeSearch
-      args: { query: "search term" }   # partial deep equality match
-    - tool: documentRetrieve
-      args: any                        # any arguments accepted
-      max_duration_ms: 5000            # per-tool latency assertion
-    - tool: summarize                  # omit args to skip argument checking
-```
-`tool-trajectory` is an AgentV-specific extension over AgentV-normalized transcripts. Do not use Promptfoo `trajectory:*`, `tool-call-f1`, `skill-used`, or `trace-*` names; AgentV rejects those until their trace semantics are implemented directly.
+### Skill And Trajectory Assertions
+```yaml
+- type: skill-used
+  value: csv-analyzer
+- type: not-skill-used
+  value:
+    pattern: "web-*"
+    max: 0
+- type: trajectory:tool-used
+  value:
+    name: knowledgeSearch
+    min: 2
+- type: trajectory:tool-sequence
+  value:
+    mode: exact
+    steps: [knowledgeSearch, documentRetrieve]
+- type: trajectory:tool-args-match
+  value:
+    name: knowledgeSearch
+    args: { query: "search term" }
+    mode: partial
+```
+Use Promptfoo-compatible `skill-used`, `not-skill-used`, and `trajectory:*`
+assertions for new eval YAML. Do not author `skill-trigger` or
+`tool-trajectory`; AgentV rejects them with migration guidance. Per-tool
+latency checks from old `tool-trajectory` YAML do not have a Promptfoo
+trajectory equivalent yet; use a `script` assertion for that behavior.
 
 ### field-accuracy
 ```yaml

package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js → ts-eval-loader-DQDYRULE-V3377FOL.js}

@@ -2,7 +2,7 @@ import { createRequire } from 'node:module'; const require = createRequire(impor
 import {
   loadTsEvalFile,
   loadTsEvalSuite
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-6ZZCDZPD.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
@@ -11,4 +11,4 @@ export {
   loadTsEvalFile,
   loadTsEvalSuite
 };
-//# sourceMappingURL=ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map
+//# sourceMappingURL=ts-eval-loader-DQDYRULE-V3377FOL.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "5.2.0-next.1",
+  "version": "5.3.0-next.1",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {