npm - agentv - Versions diffs - 5.2.0-next.1 → 5.3.1-next.1 - Mend

agentv 5.2.0-next.1 → 5.3.1-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (46) hide show

package/dist/dashboard/index.html CHANGED Viewed

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>AgentV</title>
-    <script type="module" crossorigin src="/assets/index-r_jSJmlw.js"></script>
+    <script type="module" crossorigin src="/assets/index-DTA6-l7q.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D_bokML8.css">
   </head>
   <body class="bg-gray-950 text-gray-100">

package/dist/{dist-DEPJOMCA.js → dist-NMXMI5SK.js} RENAMED Viewed

@@ -26,6 +26,7 @@ import {
   getResultsRepoSyncStatus,
   listGitRuns,
   listGitRunsCached,
+  loadEnvPathFiles,
   loadTsConfig,
   materializeGitRun,
   normalizeResultsConfig,
@@ -34,6 +35,8 @@ import {
   parseCodexSession,
   parseCopilotEvents,
   parseEnvOutput,
+  parseJsonEnv,
+  parseShellExportsEnv,
   prepareEvalWorkspace,
   prepareResultsRepoBranch,
   pushResultsRepoBranch,
@@ -44,6 +47,7 @@ import {
   resolveResultsRepoRunsDir,
   resolveResultsRepoUrl,
   runBeforeSessionHook,
+  runEnvFromEntries,
   setupWipWorktree,
   stageResultsArtifacts,
   syncProject,
@@ -54,7 +58,7 @@ import {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-ALVZQUZP.js";
+} from "./chunk-LXBI3SPX.js";
 import {
   AGENTV_CONFIG_FILE_NAME,
   AGENTV_CONFIG_YML_FILE_NAME,
@@ -83,6 +87,7 @@ import {
   ExternalTraceMetadataWireSchema,
   FieldAccuracyGrader,
   GraderRegistry,
+  LEGACY_METRICS_SCHEMA_VERSION,
   LatencyGrader,
   LlmGrader,
   METRICS_JSON_MEDIA_TYPE,
@@ -106,6 +111,7 @@ import {
   ResultRowSchemaError,
   ScriptGrader,
   SkillTriggerGrader,
+  SkillUsedGrader,
   TEST_MESSAGE_ROLES,
   TRACE_EVENT_TYPES,
   TRACE_REDACTION_LEVELS,
@@ -145,6 +151,7 @@ import {
   TraceSourceWireSchema,
   TraceSummaryBoundarySchema,
   TraceToolWireSchema,
+  TrajectoryGrader,
   WorkspaceCreationError,
   addProject,
   aggregateRunDir,
@@ -216,7 +223,7 @@ import {
   extractFailOnError,
   extractImageBlocks,
   extractJsonBlob,
-  extractLastAssistantContent,
+  extractLastAssistantContent2,
   extractTargetFromSuite,
   extractTargetRefsFromSuite,
   extractTargetsFromSuite,
@@ -263,6 +270,7 @@ import {
   isPlainConfigObject,
   isTestMessage,
   isTestMessageRole,
+  isTypeScriptEvalConfigFileName,
   killAllTrackedChildren,
   listTargetNames,
   loadConfig,
@@ -282,6 +290,7 @@ import {
   normalizeExperimentConfig,
   normalizeExperimentRunOverride,
   normalizeLineEndings,
+  normalizeMetricsArtifactWire,
   normalizeRepoIdentity,
   normalizeResultRow,
   omitExternalTraceMetadataKeys,
@@ -369,11 +378,12 @@ import {
   trackChild,
   trackedChildCount,
   transcriptReplayRecordToProviderResponse,
+  typeScriptEvalConfigGlob,
   writeArtifacts,
   writeArtifactsFromResults,
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import {
   DockerWorkspaceProvider
@@ -409,6 +419,7 @@ export {
   ExternalTraceMetadataWireSchema,
   FieldAccuracyGrader,
   GraderRegistry,
+  LEGACY_METRICS_SCHEMA_VERSION,
   LatencyGrader,
   LlmGrader,
   METRICS_JSON_MEDIA_TYPE,
@@ -433,6 +444,7 @@ export {
   RunBudgetTracker,
   ScriptGrader,
   SkillTriggerGrader,
+  SkillUsedGrader,
   TEST_MESSAGE_ROLES,
   TRACE_EVENT_TYPES,
   TRACE_REDACTION_LEVELS,
@@ -473,6 +485,7 @@ export {
   TraceSourceWireSchema,
   TraceSummaryBoundarySchema,
   TraceToolWireSchema,
+  TrajectoryGrader,
   TranscriptProvider,
   WorkspaceCreationError,
   addProject,
@@ -561,7 +574,7 @@ export {
   extractFailOnError,
   extractImageBlocks,
   extractJsonBlob,
-  extractLastAssistantContent,
+  extractLastAssistantContent2 as extractLastAssistantContent,
   extractTargetFromSuite,
   extractTargetRefsFromSuite,
   extractTargetsFromSuite,
@@ -613,11 +626,13 @@ export {
   isPlainConfigObject,
   isTestMessage,
   isTestMessageRole,
+  isTypeScriptEvalConfigFileName,
   killAllTrackedChildren,
   listGitRuns,
   listGitRunsCached,
   listTargetNames,
   loadConfig,
+  loadEnvPathFiles,
   loadEvalCaseById,
   loadEvalCases,
   loadEvalSuite,
@@ -636,6 +651,7 @@ export {
   normalizeExperimentConfig,
   normalizeExperimentRunOverride,
   normalizeLineEndings,
+  normalizeMetricsArtifactWire,
   normalizeRepoIdentity,
   normalizeResultRow,
   normalizeResultsConfig,
@@ -646,9 +662,11 @@ export {
   parseCopilotEvents,
   parseEnvOutput,
   parseEvaluationResultBoundary,
+  parseJsonEnv,
   parseJsonFromText,
   parseJsonSafe,
   parseJsonlResults,
+  parseShellExportsEnv,
   parseTraceBoundary,
   parseTraceSummaryBoundary,
   parseYamlValue,
@@ -687,6 +705,7 @@ export {
   runContainsAnyAssertion,
   runContainsAssertion,
   runEndsWithAssertion,
+  runEnvFromEntries,
   runEqualsAssertion,
   runEvalCase,
   runEvaluation,
@@ -749,9 +768,10 @@ export {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult,
+  typeScriptEvalConfigGlob,
   writeArtifacts,
   writeArtifactsFromResults,
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=dist-DEPJOMCA.js.map
+//# sourceMappingURL=dist-NMXMI5SK.js.map

package/dist/index.js CHANGED Viewed

@@ -6,11 +6,11 @@ import {
   runCli,
   shouldRunBeforeSessionHook,
   usesDeprecatedStudioAlias
-} from "./chunk-3X357HS4.js";
-import "./chunk-T4AD3H3Y.js";
-import "./chunk-46K2OET3.js";
-import "./chunk-ALVZQUZP.js";
-import "./chunk-3S6A2RKR.js";
+} from "./chunk-ASIGJIOJ.js";
+import "./chunk-LKGARI3W.js";
+import "./chunk-RKE7SSET.js";
+import "./chunk-LXBI3SPX.js";
+import "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";

package/dist/{interactive-36ZNMQB2.js → interactive-BN527UV3.js} RENAMED Viewed

@@ -8,14 +8,14 @@ import {
   getCategories,
   resolveExistingRunPrimaryPath,
   runEvalCommand
-} from "./chunk-T4AD3H3Y.js";
-import "./chunk-46K2OET3.js";
-import "./chunk-ALVZQUZP.js";
+} from "./chunk-LKGARI3W.js";
+import "./chunk-RKE7SSET.js";
+import "./chunk-LXBI3SPX.js";
 import {
   getAgentvConfigDir,
   listTargetNames,
   readTargetDefinitions
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
@@ -344,4 +344,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
 export {
   launchInteractiveWizard
 };
-//# sourceMappingURL=interactive-36ZNMQB2.js.map
+//# sourceMappingURL=interactive-BN527UV3.js.map

package/dist/skills/agentv-bench/SKILL.md CHANGED Viewed

@@ -49,7 +49,7 @@ Before running or optimizing, understand what you're working with.
 2. **Identify success criteria** — what does "good" look like for this agent? What are the edge cases? What would a failure look like? Talk to the user if this isn't clear from the artifacts alone.
-3. **Understand the target harness** — which provider runs the agent (Claude, GPT, Copilot CLI, Gemini, custom CLI)? This affects what grader types are available and how to run tests. Targets are configured in `.agentv/targets.yaml` (canonical location, searched from the eval file directory upward). Sensitive values like `api_key` must use `${{ ENV_VAR }}` syntax — literal secrets are rejected as a security guardrail.
+3. **Understand the target harness** — which provider runs the agent (Claude, GPT, Copilot CLI, Gemini, custom CLI)? This affects what grader types are available and how to run tests. Targets are configured in `.agentv/targets.yaml` (canonical location, searched from the eval file directory upward). Sensitive values like `api_key` must use `{{ env.ENV_VAR }}` references — literal secrets are rejected as a security guardrail.
 4. **Challenge assumptions** — if evals already exist, review their quality before running:
    - Are the test cases testing the right things?
@@ -65,7 +65,7 @@ Before running or optimizing, understand what you're working with.
 AgentV supports two evaluation formats:
-**EVAL.yaml** (native, full features) — supports environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation.
+**EVAL.yaml** (native, full features) — supports prompt matrices, environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation.
 ```yaml
 # example.eval.yaml
@@ -89,7 +89,7 @@ tests:
       - Review identifies the null pointer bug and suggests a concrete fix
 ```
-Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs.
+Multi-skill evaluation is handled naturally via prompt messages — describe the task in `prompts`, put row-specific data in `tests[].vars`, and the agent uses whatever skills it needs.
 **evals.json** (skill-creator compatible) — auto-promoted to EVAL-equivalent format:
 - `prompt` → input messages
@@ -117,7 +117,7 @@ Start with 2-3 realistic test cases — the kind of thing a real user would actu
 Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment.
-**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `assert-set`, `script`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
+**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `assert-set`, `skill-used`, `not-skill-used`, `trajectory:*`, `script`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
 Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-rubric`.
@@ -127,7 +127,7 @@ Prefer deterministic graders over LLM graders whenever possible. If an assertion
 This section is one continuous sequence — don't stop partway through.
-Each run produces a new `.agentv/results/default/<timestamp>/` directory automatically. Use timestamps to identify iterations when comparing runs.
+Each run produces a new canonical `.agentv/results/<run_id>/` directory automatically. Use the printed run id and `.internal/index.jsonl` when comparing runs.
 ### Choosing a run mode
@@ -228,11 +228,11 @@ Each grader subagent (operating under `agents/grader.md` instructions):
 2. Reads `<test-id>/response.md` for the candidate output
 3. Grades the response against the prompt criteria
 4. **Writes its result to disk**: `<run-dir>/<evalset>/<test-id>/llm_grader_results/<name>.json`
-5. Returns score (0.0–1.0) and per-assertion evidence to the orchestrator
+5. Returns score (0.0–1.0) and per-assertion notes to the orchestrator
-**Writing to disk is critical.** Assertion arrays are lost if accumulated only in the orchestrator's context across multiple batches (context summarization drops detail). Writing per-test results to `llm_grader_results/<name>.json` makes grading resumable and assertion evidence durable.
+**Writing to disk is critical.** Intermediate assertion arrays are lost if accumulated only in the orchestrator's context across multiple batches (context summarization drops detail). Writing per-test results to `llm_grader_results/<name>.json` makes grading resumable. Public `grading.json` artifacts normalize these details into recursive `component_results`.
-The result file format is:
+The intermediate subagent result file format is:
 ```json
 { "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
 ```
@@ -246,18 +246,19 @@ agentv pipeline bench <run-dir>
 agentv results validate <run-dir>
 ```
-`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with script-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `summary.json`.
+`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with script-grader scores, computes aggregate scores, and writes the canonical run bundle with `.internal/index.jsonl`, `summary.json`, and per-attempt `grading.json`.
 > **Diagnosing `pass_rate=0`:** If `pipeline bench` reports `pass_rate=0` across the board, do **not** assume the tests genuinely failed. First verify the grading pipeline ran correctly: check that `<test-id>/llm_grader_results/<name>.json` exists and is non-empty for each test. If these files are absent or empty, the grader subagents failed to produce output (most common cause: `agents/grader.md` was not embedded in the subagent prompts — see Phase 2). Treat `pass_rate=0` as a real signal only after confirming grader results exist.
 ### Artifacts
 All artifacts use established schemas — see `references/schemas.md` for the full definitions. Do not modify the structure. Key artifacts per run:
-- **grading.json**: per-test assertions with `{text, passed, evidence}`, plus summary
-- **timing.json**: `{total_tokens, duration_ms, total_duration_seconds}`
-- **summary.json**: per-target aggregate `{pass_rate, time_seconds, tokens}`
+- **.internal/index.jsonl**: canonical row index for compare, Dashboard, rerun, and artifact discovery
+- **grading.json**: aggregate `pass`, `score`, `reason`, recursive `component_results`, and optional `assertion`, `named_scores`, and `metadata`
+- **metrics.json**: token, cost, timing, tool, and transcript metrics for the attempt
+- **summary.json**: run-level aggregate metadata and pass/score rollups
-Write artifacts to `.agentv/artifacts/` or the iteration directory.
+Write artifacts to `.agentv/results/<run_id>/` through the CLI unless a task explicitly needs a fixed output path.
 ### Environment features (EVAL.yaml only)
@@ -350,7 +351,7 @@ If a prompt file is referenced in both task input and grader configs, optimize f
 After improving:
 1. Apply your changes to the agent's prompts/skills/config
-2. Re-run all test cases (agentv creates a new `.agentv/results/default/<timestamp>/` directory automatically)
+2. Re-run all test cases (agentv creates a new `.agentv/results/<run_id>/` directory automatically)
 3. Compare against the previous iteration (Step 4). If running in automated mode, use the **automated keep/discard** logic below instead of manual judgment — it will decide whether to keep or revert the change for you.
 4. Present results to the user (or log the decision if running automated keep/discard)
 5. Stop when ANY of:

package/dist/skills/agentv-bench/agents/analyzer.md CHANGED Viewed

@@ -22,7 +22,7 @@ You are an eval-quality analyst for AgentV. Your job is to read JSONL evaluation
 Read every line of the JSONL results file. Each line is a JSON object with:
 - `test_id`, `suite`, `score`, `assertions`, `reasoning`, `target`
-- `scores` (optional): Array of per-grader breakdowns with `name`, `type`, `score`, `weight`, `verdict`, `assertions`, `reasoning`
+- `scores` (optional): Array of per-grader breakdowns with `name`, `type`, `score`, `weight`, `pass`, `assertions`, `reasoning`
 If `eval-path` is provided, also read the EVAL.yaml to understand grader configurations.
@@ -123,7 +123,7 @@ If a section has no findings, include the header with "None found." underneath.
 - **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config.
 - **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on.
 - **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-rubric` → deterministic saves the most).
-- **Handle all grader types:** Process `script`, `tool-trajectory`, `llm-rubric`, `rubrics`, `assert-set`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
+- **Handle all grader types:** Process `script`, `trajectory:*`, `skill-used`, `not-skill-used`, `llm-rubric`, `rubrics`, `assert-set`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
 - **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific.
 - **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning.

package/dist/skills/agentv-bench/agents/comparator.md CHANGED Viewed

@@ -24,7 +24,7 @@ You are the Blind Comparator for AgentV's evaluation workflow. Your job is to co
 You will receive:
 - `outputs`: Array of evaluation outputs to compare. Each contains:
   - `target_id`: The provider/configuration identifier (DO NOT read this during scoring)
-  - `evaluator_results`: Array of grader scores and details (script, tool-trajectory, llm-rubric, deterministic)
+  - `evaluator_results`: Array of grader scores and details (script, trajectory, skill-use, llm-rubric, deterministic)
   - `workspace_changes`: File changes made during workspace evaluation (if applicable)
   - `tool_calls`: Tool invocations and results from multi-turn conversations (if applicable)
   - `conversation`: Full multi-turn conversation history (if applicable)
@@ -91,7 +91,7 @@ For each content criterion, define:
 **Grader-Specific Scoring** — when grader results are present:
 - **script-grader**: Factor in pass/fail results, test coverage, assertion hit rates
-- **tool-trajectory**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
+- **trajectory assertions**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
 - **llm-rubric**: Factor in existing LLM grader scores as a reference signal (not as ground truth)
 - **deterministic**: Factor in exact match / keyword hit rates
@@ -235,7 +235,7 @@ Also produce a human-readable markdown summary:
 - **Be evidence-based**: Every score must cite specific evidence from the output.
 - **Evaluate substance over style**: Correct, complete answers with rough formatting score higher than polished but incorrect answers.
 - **Handle missing data gracefully**: If an output lacks workspace changes or tool calls but others have them, score what is present — do not penalize for data the target wasn't expected to produce.
-- **Respect grader signals**: When script-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily.
+- **Respect grader signals**: When script-grader or trajectory assertion results exist, they represent objective ground truth. Weight these heavily.
 ## Edge Cases

package/dist/skills/agentv-bench/agents/executor.md CHANGED Viewed

@@ -14,7 +14,7 @@ You are the executor for an AgentV evaluation test case. Your job is to **perfor
 You are the target agent being evaluated. Do the task to the best of your ability — your output will be graded by a separate grader agent.
 **You will receive these parameters:**
-- `test-dir`: Path to the test case directory (e.g., `.agentv/results/default/<timestamp>/<test-id>/`)
+- `test-dir`: Path to the prepared test case directory inside the run bundle (for example `.agentv/results/<run_id>/<result-dir>/`)
 ## Process

package/dist/skills/agentv-bench/agents/grader.md CHANGED Viewed

@@ -80,8 +80,10 @@ For LLM-graded types: be rigorous and fair. Score based on substance, not exact
 | Type | How to evaluate |
 |------|----------------|
-| `tool-trajectory` | Inspect transcript for tool calls, match against expected sequence/mode |
-| `skill-trigger` | Check if the named skill was invoked in tool calls |
+| `trajectory:tool-used` | Inspect transcript for tool presence and minimum call counts |
+| `trajectory:tool-sequence` | Inspect transcript for expected tool call order |
+| `trajectory:tool-args-match` | Inspect transcript for expected tool arguments |
+| `skill-used` / `not-skill-used` | Check whether the named skill was invoked in tool calls |
 If transcript data is not available for tool inspection assertions, record `score: null` with a note that transcript data was not captured. Exclude from the weighted average.

package/dist/skills/agentv-bench/references/autoresearch.md CHANGED Viewed

@@ -56,14 +56,14 @@ Before proceeding to the next iteration, log the decision and rationale so the u
 Iteration 2: KEEP
   wins=3, losses=1, ties=6, meanDelta=+0.05
   Rationale: candidate wins outweigh losses (3 > 1)
-  Baseline promoted: .agentv/results/default/20250101-120000/index.jsonl
+  Baseline promoted: .agentv/results/2025-01-01T12-00-00-000Z/.internal/index.jsonl
 ```
 ```
 Iteration 3: DISCARD
   wins=1, losses=2, ties=7, meanDelta=-0.03
   Rationale: candidate losses outweigh wins (2 > 1)
-  Reverted to baseline: .agentv/results/default/20250101-110000/index.jsonl
+  Reverted to baseline: .agentv/results/2025-01-01T11-00-00-000Z/.internal/index.jsonl
   Next: try a different mutation
 ```
@@ -210,20 +210,20 @@ agentv eval <eval-path> --experiment autoresearch-<name>
 **b. Extract scores (bash only — do NOT read result files into your context)**
-Find the latest timestamped directory in the experiment folder. Use bash/jq to extract small structured values:
+Find the latest run directory. Use bash/jq to extract small structured values:
 ```bash
 # Find latest run dir
-RUN_DIR=$(ls -td <experiment-dir>/20*/ | head -1)
+RUN_DIR=$(ls -td .agentv/results/20*/ | head -1)
 # Overall score (mean of all scores in index.jsonl)
-SCORE=$(jq -sr '[.[].scores[].score] | add / length' "$RUN_DIR/index.jsonl")
+SCORE=$(jq -sr '[.[].score] | add / length' "$RUN_DIR/.internal/index.jsonl")
-# Per-assertion pass rates as JSON object
-PASS_RATES=$(jq -sr '[.[].scores[]] | group_by(.type) | map({key: .[0].type, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/index.jsonl")
+# Per-target mean scores as JSON object
+TARGET_SCORES=$(jq -sr 'group_by(.target) | map({key: .[0].target, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/.internal/index.jsonl")
-# Cost (if timing.json exists)
-COST=$(jq -r '.cost_usd // 0' "$RUN_DIR/timing.json" 2>/dev/null || echo 0)
+# Cost (if present in summary.json)
+COST=$(jq -r '.cost_usd // .total_cost_usd // 0' "$RUN_DIR/summary.json" 2>/dev/null || echo 0)
 ```
 Capture only these small outputs (`SCORE`, `PASS_RATES`, `COST`) — never read the full JSONL into context.

package/dist/skills/agentv-bench/references/description-optimization.md CHANGED Viewed

@@ -6,7 +6,7 @@ core workflow step.
 **Provider compatibility**: Description optimization applies to any agent platform with
 skill-discovery mechanisms — Claude Code, Codex (`.agents/` or `.codex/` folders), Copilot,
-and others. The `skill-trigger` grader checks whether the agent invoked the right skill,
+and others. The `skill-used` and `not-skill-used` graders check whether the agent invoked the right skill,
 regardless of how discovery works on that platform.
 ## Step 1: Generate Trigger EVAL.yaml
@@ -36,14 +36,13 @@ tests:
   - id: should-trigger-casual-optimize
     input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it"
     assert:
-      - type: skill-trigger
-        skill: agentv-bench
+      - type: skill-used
+        value: agentv-bench
   - id: should-not-trigger-build-error
     input: "my TypeScript build is failing with type errors in src/auth.ts"
     assert:
-      - type: skill-trigger
-        skill: agentv-bench
-        should_trigger: false
+      - type: not-skill-used
+        value: agentv-bench
 ```
 ## Step 2: Review with User

package/dist/skills/agentv-bench/references/environment-adaptation.md CHANGED Viewed

@@ -21,14 +21,14 @@ for each test case, show the prompt and output. Ask for feedback inline. Skip be
 - **Codex**: Supports skills via `.agents/` or `.codex/` folders. Emits `command_execution`
   and `file_change` tool calls.
 - **Custom CLI**: Needs `command` and output file pattern in target config
-- **Target config**: Uses `${{ ENV_VAR }}` syntax (not `${ENV_VAR}`) for API keys
+- **Target config**: Uses `{{ env.ENV_VAR }}` references for API keys
 **Note**: "Description Optimization" (see `references/description-optimization.md`) applies
 to any platform with skill-discovery mechanisms. All listed providers support skills.
 ## Unsupported Providers: Use a Code-Grader
-The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out
+The built-in `skill-used` and `not-skill-used` graders cover Claude, Copilot, Pi, Codex and VS Code out
 of the box. For providers with different tool-call formats, write a script-grader that inspects
 the agent's transcript messages or tool call trace.
@@ -37,17 +37,17 @@ transcript `messages`, and structured `trace`. Inspect `messages` or `trace.even
 tool calls; reserve `output` for final-answer text checks.
 ```yaml
-# Example: script-grader for Codex skill-trigger detection
+# Example: script-grader for Codex skill-use detection
 tests:
-  - id: should-trigger-codex
+  - id: codex-skill-use
     input: "Analyze this CSV file"
     assert:
       - type: script
-        command: [bun, run, ./judges/codex-skill-trigger.ts]
+        command: [bun, run, ./judges/codex-skill-use.ts]
 ```
 ```typescript
-// judges/codex-skill-trigger.ts
+// judges/codex-skill-use.ts
 import { defineScriptGrader } from '@agentv/sdk';
 export default defineScriptGrader(({ messages }) => {