@alpic-ai/tracks 0.5.0 → 0.6.0

package/skills/agent-experience-evals/evals/evals.json

@@ -0,0 +1,90 @@
+{
+  "skill_name": "agent-experience-evals",
+  "evals": [
+    {
+      "id": 1,
+      "name": "plan-before-spending",
+      "prompt": "Use the supplied example skill to plan its tests for two repeats with a $1 per-run budget and a 600-second time limit. Show me the case/model combinations and maximum Run budgets. Do not start paid Runs yet.",
+      "expected_output": "The Agent executes the packaged planner against the example input, inspects plan.json, reports two independent Runs and $2 maximum Run budgets excluding judge costs, and does not run or grade the plan.",
+      "files": ["evals/files/README.md", "evals/files/example/"],
+      "assertions": [
+        {
+          "id": "uses-planner",
+          "text": "Executes scripts/evals.mjs plan and inspects the produced plan.json rather than inventing a plan."
+        },
+        { "id": "independent-repeats", "text": "Reports two distinct variants with repeats 1 and 2." },
+        {
+          "id": "budget-explained",
+          "text": "Reports $2 maximum total Run budgets, distinguishing it from judge costs and actual spend."
+        },
+        { "id": "no-paid-run", "text": "Does not start Runs or contact System One." }
+      ]
+    },
+    {
+      "id": 2,
+      "name": "configure-before-running",
+      "prompt": "Plan the example skill for three repeats. Add a setup script that checks node --version, a teardown script containing true, and MCP server tools=https://mcp.example.com. Budget is $1 per Run, time limit is 600 seconds. Show the saved configuration and total Run count; explain any account connection needed before execution. Only plan: do not create connections or start Runs.",
+      "expected_output": "The Agent runs the packaged planner, inspects saved setup/teardown/MCP configuration, reports three Runs and $3 budget ceiling excluding judging, and explains authenticated MCP account access separately from planning without creating a connection.",
+      "files": ["evals/files/README.md", "evals/files/example/"],
+      "assertions": [
+        {
+          "id": "saved-environment",
+          "text": "Verifies setup, teardown and MCP settings in plan.json match the request."
+        },
+        { "id": "correct-count", "text": "Reports three Runs and $3 maximum Run budgets, excluding judge cost." },
+        {
+          "id": "account-prerequisite",
+          "text": "Explains account sign-in must be completed and active if the MCP requires it; a URL alone does not provide authentication."
+        },
+        { "id": "no-unrequested-work", "text": "Does not create an MCP connection, start a Run or grade the plan." }
+      ]
+    },
+    {
+      "id": 3,
+      "name": "report-unstarted-comparison",
+      "prompt": "Plan the example skill for two repeats, but do not start them. Use the packaged grade command to report the unstarted comparison, then inspect its JSON summary and HTML report. No TypeSafe key is available. Explain whether any work passed, and regenerate the report without running agents or contacting a judge.",
+      "expected_output": "The Agent executes planning and grading, verifies both unstarted cases appear as incomplete/not_started with no graded checks, produces and regenerates report.html, and accurately reports that no Runs executed or checks passed.",
+      "files": ["evals/files/README.md", "evals/files/example/"],
+      "assertions": [
+        {
+          "id": "real-report",
+          "text": "Executes grade and report commands and inspects summary.json and report.html."
+        },
+        { "id": "unstarted-visible", "text": "Reports two incomplete/not_started cases, with no recording available." },
+        {
+          "id": "no-false-passes",
+          "text": "Does not count unstarted cases as passing checks or claim an evaluated outcome."
+        },
+        {
+          "id": "no-external-work",
+          "text": "Does not start Runs or call System One; explains no judge key is needed to report unstarted cases."
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "name": "missing-runner-credentials",
+      "prompt": "Plan the example skill for one repeat, then attempt the real run command with AGENT_RUNNER_API_KEY unset. Tell me what actually happens and the smallest next step. No API credentials are available; do not invent them, substitute a fake CLI, or claim the task ran.",
+      "expected_output": "The Agent executes the planner and real runner, verifies the missing API key prevents creation, explains how to configure Agent Runner authentication using the linked guide, and preserves the plan for a later retry without claiming a Run executed.",
+      "files": ["evals/files/README.md", "evals/files/example/"],
+      "assertions": [
+        {
+          "id": "attempted-real-cli",
+          "text": "Attempts the packaged run command with the actual agr CLI and API key unset, recording the failure."
+        },
+        {
+          "id": "accurate-blocker",
+          "text": "Identifies missing Agent Runner authentication, not a model failure, as the blocker."
+        },
+        {
+          "id": "honest-result",
+          "text": "Does not invent a Run ID, grade or successful execution, or replace agr with a fake."
+        },
+        {
+          "id": "retryable-plan",
+          "text": "Preserves the plan and directs the user to configure Agent Runner credentials before retrying."
+        }
+      ]
+    }
+  ]
+}

package/skills/agent-experience-evals/evals/files/README.md

@@ -0,0 +1,7 @@
+# Evaluation inputs
+
+`example` is a minimal skill with one case and one model target for planning exercises. It contains no canned Run recordings or judge responses.
+
+The merge workflow places the skill in `skills/agent-experience-evals` inside a git checkout. It installs the real `agr` CLI in `/workspace/.eval-tools/node_modules/.bin`; prepend that directory to PATH when running commands. Do not pass the outer workflow's API or judge credentials to the tested Agent.
+
+These tasks are executed by real agents. They request planning and prerequisite/report handling, not recursive paid evaluations of the input skill.

package/skills/agent-experience-evals/evals/files/example/SKILL.md

@@ -0,0 +1,6 @@
+---
+name: fixture-example
+description: A minimal input skill for agent-experience-evals evaluations.
+---
+
+Respond to the supplied prompt. Its eval is a planning input; only start its Runs when explicitly requested.

package/skills/agent-experience-evals/evals/files/example/evals/evals.json

@@ -0,0 +1,12 @@
+{
+  "skill_name": "fixture-example",
+  "evals": [
+    {
+      "id": 1,
+      "name": "example",
+      "prompt": "Perform the fixture task.",
+      "expected_output": "The fixture task was completed correctly.",
+      "assertions": ["The recorded output demonstrates correct work."]
+    }
+  ]
+}

package/skills/agent-experience-evals/evals/files/example/evals/targets.json

@@ -0,0 +1,3 @@
+{
+  "claude": ["claude-sonnet-4-6"]
+}

package/skills/agent-experience-evals/evals/targets.json

@@ -0,0 +1,4 @@
+{
+  "claude": ["claude-sonnet-4-6"],
+  "codex": ["gpt-5.4"]
+}

package/skills/agent-experience-evals/report.html

@@ -0,0 +1,149 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Agent experience evals</title>
+    <style>
+      :root {
+        color-scheme: light;
+        font-family: system-ui, sans-serif;
+        color: #17213a;
+        background: #f4f6fa;
+      }
+      body {
+        max-width: 1280px;
+        margin: 36px auto;
+        padding: 0 24px;
+      }
+      h1 {
+        margin-bottom: 8px;
+      }
+      p {
+        line-height: 1.5;
+      }
+      .metrics {
+        display: flex;
+        gap: 12px;
+        flex-wrap: wrap;
+        margin: 24px 0;
+      }
+      .metric {
+        background: white;
+        border: 1px solid #dae0eb;
+        border-radius: 8px;
+        padding: 16px;
+        min-width: 110px;
+      }
+      .metric strong {
+        display: block;
+        font-size: 28px;
+        margin-top: 10px;
+      }
+      .badge {
+        display: inline-block;
+        padding: 3px 9px;
+        border-radius: 5px;
+        font-size: 13px;
+        font-weight: 600;
+      }
+      .passed {
+        background: #dcfce7;
+        color: #14532d;
+      }
+      .failed {
+        background: #fee2e2;
+        color: #7f1d1d;
+      }
+      .blocked {
+        background: #fef3c7;
+        color: #78350f;
+      }
+      .inconclusive {
+        background: #e0e7ff;
+        color: #312e81;
+      }
+      .incomplete,
+      .neutral {
+        background: #e5e7eb;
+        color: #374151;
+      }
+      .table-wrap {
+        overflow-x: auto;
+        background: white;
+        border: 1px solid #dae0eb;
+        border-radius: 8px;
+      }
+      table {
+        width: 100%;
+        border-collapse: collapse;
+        font-size: 14px;
+      }
+      th,
+      td {
+        text-align: left;
+        padding: 14px;
+        border-bottom: 1px solid #e5e7eb;
+        vertical-align: top;
+        overflow-wrap: anywhere;
+      }
+      th {
+        background: #f8fafc;
+      }
+      small {
+        color: #526078;
+      }
+      .detail td {
+        padding-top: 0;
+      }
+      details {
+        padding: 10px 0;
+      }
+      summary {
+        cursor: pointer;
+        color: #234fb0;
+      }
+      a {
+        color: #234fb0;
+      }
+      footer {
+        color: #526078;
+        margin: 24px 0;
+      }
+    </style>
+  </head>
+  <body>
+    <h1>Agent experience evals</h1>
+    <p>
+      Run completion and test success are separate. Expand each case to inspect its expected outcome and individual
+      checks.
+    </p>
+    <p><strong>Expected outcomes:</strong> {{OUTCOMES}}</p>
+    <h2>Assertion checks</h2>
+    <div class="metrics">{{METRICS}}</div>
+    <p>
+      <strong>Conclusive assertion pass rate: {{RATE}}</strong> · Incomplete runs: {{INCOMPLETE}}<br />The rate excludes
+      blocked and inconclusive checks, expected-outcome checks, and incomplete runs. These remain visible below.
+    </p>
+    <div class="table-wrap">
+      <table>
+        <thead>
+          <tr>
+            <th>Case</th>
+            <th>Harness / model</th>
+            <th>Run completion</th>
+            <th>Expected outcome</th>
+            <th>Assertions passed / graded</th>
+          </tr>
+        </thead>
+        <tbody>
+          {{ROWS}}
+        </tbody>
+      </table>
+    </div>
+    <footer>
+      Fixed rubric, AI judgement. Confidence is not proof of correctness. Evidence links work when this page stays
+      beside its result directories. Recordings may contain sensitive data; review before sharing.
+    </footer>
+  </body>
+</html>