kushi-agents 5.0.2 → 5.0.4

package/plugin/skills/build-state/evals/evals.json

@@ -0,0 +1,31 @@
+{
+  "skill": "build-state",
+  "version": "1.0.0",
+  "description": "Karpathy State layout — index.md + log.md + per-category folders.",
+  "cases": [
+    {
+      "id": "bs-state-index",
+      "name": "fixture State/index.md has kushi_state_page front-matter",
+      "input": "validate state fixture",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "args": { "read_fixture": "State/index.md" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "kushi_state_page:\\s*true" }
+      ]
+    },
+    {
+      "id": "bs-state-log-exists",
+      "name": "fixture State/log.md exists",
+      "input": "verify log",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "State/log.md" },
+        { "type": "file-exists", "path": "State/index.md" }
+      ]
+    }
+  ]
+}

package/plugin/skills/consolidate-evidence/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "consolidate-evidence",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for consolidate-evidence. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "consolidate-evidence-smoke-1",
+      "name": "consolidate-evidence produces a non-empty response",
+      "input": "synthetic consolidate-evidence probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "consolidate-evidence-smoke-2",
+      "name": "consolidate-evidence echoes case id",
+      "input": "case-id consolidate-evidence-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "consolidate-evidence-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/dashboard/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "dashboard",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for dashboard. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "dashboard-smoke-1",
+      "name": "dashboard produces a non-empty response",
+      "input": "synthetic dashboard probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "dashboard-smoke-2",
+      "name": "dashboard echoes case id",
+      "input": "case-id dashboard-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "dashboard-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/emit-vertex/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "emit-vertex",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for emit-vertex. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "emit-vertex-smoke-1",
+      "name": "emit-vertex produces a non-empty response",
+      "input": "synthetic emit-vertex probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "emit-vertex-smoke-2",
+      "name": "emit-vertex echoes case id",
+      "input": "case-id emit-vertex-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "emit-vertex-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/eval/SKILL.md

@@ -0,0 +1,90 @@
+---
+name: "eval"
+version: "1.0.0"
+description: "USE WHEN the user says \"run evals\", \"eval canary\", \"eval ask-project\", \"check skill regression\", \"update eval baseline\", or before tagging a release. DO NOT USE for evidence validation of a real project (use ask-project / project-status). Capability: runs per-skill evals (deterministic script graders + optional LLM-rubric graders) from each skill's evals/evals.json, aggregates pass-rate / latency / token metrics, and compares against evals/baseline.json to flag regressions. Synthetic fixtures only — never live customer data."
+---
+
+# Skill: eval
+
+The objective regression signal for every other kushi skill. Spec: <https://agentskills.io/skill-creation/evaluating-skills>. Doctrine: `instructions/skill-evals.instructions.md`.
+
+User triggers: "run evals", "eval canary", "eval <skill>", "eval all", "update eval baseline", "check skill regression".
+
+## USE WHEN
+
+- About to commit a behavioral change to any `plugin/skills/<name>/SKILL.md` or its companion script.
+- Reviewing a PR that touches skills, prompts, or shared instructions.
+- Tagging a release (full suite is implied by the release script).
+- Investigating "did this skill get worse?" after a refactor.
+
+## DO NOT USE FOR
+
+- Validating real customer evidence (use `ask-project` / `project-status`).
+- Live M365 / ADO / CRM probes (use `pull-*` directly).
+- Generating new fixtures from real engagements — that violates the privacy rule.
+
+## Gotchas
+
+1. **Canary ≠ full.** `npm run eval:canary` runs ~6 skills. Use `npm run eval:all` before tagging.
+2. **Synthetic fixtures only.** Never copy real customer evidence into `evals/fixtures/`. The runner does NOT enforce this — humans do.
+3. **`pull-*` skills run in `--cached` / `--dry-run`.** Live network calls are explicitly disabled in canary mode; the LLM-rubric subset is skipped unless `-Live` is passed.
+4. **Baseline drift is OK after intentional changes.** Refresh with `npm run eval:baseline`. NEVER auto-update baseline in CI.
+5. **`Evidence/_evals/` is gitignored.** If you need to share a run, copy the JSON manually.
+6. **Tokens are estimates.** When a grader can't measure tokens (e.g. a pwsh-only run), it records `0` and the aggregator excludes those from the mean.
+
+## Step checklist
+
+- [ ] Pick mode: `-Skill <name>` (one skill) · `-Canary` (fast subset) · `-All` (full suite).
+- [ ] Confirm `evals/baseline.json` exists; if missing, run with `-UpdateBaseline` first.
+- [ ] Run: `pwsh plugin/skills/eval/run-evals.ps1 -Canary` (or other mode).
+- [ ] Inspect the output JSON under `Evidence/_evals/` and the `benchmark.json` summary.
+- [ ] If regressions flagged: re-run the specific skill with `-Skill <name>` for detail.
+- [ ] If intentional change: bump the skill version, then `npm run eval:baseline`.
+
+## Validation loop
+
+After running, verify:
+
+- Output JSON exists at the path printed by the runner.
+- Every case has `pass: true|false` AND `duration_ms` AND a per-assertion breakdown.
+- `benchmark.json` has `summary.regressions` array (empty if clean).
+- Exit code: `0` = all green vs baseline; `1` = at least one regression.
+
+If a case errors out before any assertion runs (e.g. fixture missing), it counts as `pass: false` with `error: "<message>"` so totals stay honest.
+
+## What the runner does (`run-evals.ps1`)
+
+1. Discovers `plugin/skills/<name>/evals/evals.json` for the requested skills.
+2. Validates each file against `plugin/skills/eval/evals.schema.json`.
+3. For each case:
+   - Resolves the fixture path (`fixture` field, relative to repo root).
+   - Dispatches per `grader_type`:
+     - `script` → invokes the skill's known executable (`run.ps1`, a `*.mjs` probe, or a deterministic shim under `evals/probe.*`) with the case input.
+     - `llm` → only when `-Live` is set; otherwise marked `skipped`.
+   - Captures `stdout`, `stderr`, `duration_ms`.
+   - Runs every assertion in `expected_assertions`; collects per-assertion pass/fail.
+   - Case `pass` = all assertions pass AND no error.
+4. Writes per-run JSON to `Evidence/_evals/<utc-timestamp>.json` (or `-Output` override).
+5. Calls `node src/eval-aggregator.mjs` to compute `benchmark.json` and compare against `evals/baseline.json`.
+6. Prints a one-screen summary; exits 0 on clean, 1 on regression.
+
+## Arguments
+
+| Flag | Purpose |
+|---|---|
+| `-Skill <name>` | Run only one skill's evals. |
+| `-All` | Run every `plugin/skills/<name>/evals/evals.json`. |
+| `-Canary` | Run only cases marked `"canary": true`. |
+| `-Output <path>` | Override per-run JSON path. Default `Evidence/_evals/<ts>.json`. |
+| `-Baseline` | Compare against `evals/baseline.json` (default ON). |
+| `-UpdateBaseline` | Write current run's metrics into `evals/baseline.json`. |
+| `-Live` | Allow LLM-rubric cases (requires `m_*` tools / sub-agent). |
+| `-StrictExit` | Exit 1 on any case failure (CI mode). |
+
+## References
+
+- `plugin/instructions/skill-evals.instructions.md` (doctrine)
+- `plugin/instructions/agentskills-compliance.instructions.md` (size + section caps for SKILL.md)
+- `src/eval-aggregator.mjs` (mean/stddev + regression detection)
+- `evals/baseline.json` (committed baseline)
+- <https://agentskills.io/skill-creation/evaluating-skills>

package/plugin/skills/eval/evals.schema.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://gim-home.github.io/kushi/schemas/evals.schema.json",
+  "title": "Kushi per-skill evals file",
+  "description": "Schema for plugin/skills/<name>/evals/evals.json. Validated by self-check D33.evals-schema.",
+  "type": "object",
+  "required": ["skill", "cases"],
+  "additionalProperties": false,
+  "properties": {
+    "skill": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Skill name (must match the parent directory and SKILL.md frontmatter name)."
+    },
+    "version": { "type": "string" },
+    "description": { "type": "string" },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/case" }
+    }
+  },
+  "definitions": {
+    "case": {
+      "type": "object",
+      "required": ["id", "name", "input", "expected_assertions", "grader_type"],
+      "additionalProperties": false,
+      "properties": {
+        "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" },
+        "name": { "type": "string", "minLength": 1 },
+        "input": {
+          "anyOf": [{ "type": "string" }, { "type": "object" }]
+        },
+        "fixture": { "type": "string" },
+        "canary": { "type": "boolean", "default": false },
+        "skip": { "type": "boolean", "default": false },
+        "skip_reason": { "type": "string" },
+        "timeout_ms": { "type": "integer", "minimum": 100 },
+        "args": { "type": "object" },
+        "grader_type": { "enum": ["script", "llm"] },
+        "expected_assertions": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/definitions/assertion" }
+        }
+      }
+    },
+    "assertion": {
+      "type": "object",
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "enum": [
+            "file-exists",
+            "file-contains",
+            "json-path-equals",
+            "regex-match",
+            "llm-rubric"
+          ]
+        },
+        "path": { "type": "string" },
+        "needle": { "type": "string" },
+        "pattern": { "type": "string" },
+        "flags": { "type": "string" },
+        "json_path": { "type": "string" },
+        "equals": {},
+        "rubric": { "type": "string" },
+        "min_score": { "type": "number" }
+      },
+      "additionalProperties": false
+    }
+  }
+}

package/plugin/skills/eval/run-evals.ps1

@@ -0,0 +1,372 @@
+<#
+.SYNOPSIS
+Kushi skill evals runner (v5.0.3+).
+
+.DESCRIPTION
+Discovers per-skill evals at `plugin/skills/<name>/evals/evals.json`,
+validates against `plugin/skills/eval/evals.schema.json`, dispatches each case,
+runs the assertions, and writes a per-run JSON to `Evidence/_evals/<ts>.json`.
+
+Per the doctrine in `plugin/instructions/skill-evals.instructions.md`, the
+runner NEVER makes live M365 / ADO / CRM calls. Skills that genuinely require
+live calls (`pull-*`) ship cached fixtures.
+
+Dispatch:
+  - `grader_type: "script"` cases are dispatched to:
+      1. `plugin/skills/<skill>/evals/probe.ps1`  (preferred, per-skill probe)
+      2. `plugin/skills/<skill>/evals/probe.mjs`  (Node probe)
+      3. The built-in "fixture-echo" probe, which reads `case.args.read_fixture`
+         and emits its content. Lets simple read-only assertions work with no
+         per-skill code at all.
+  - `grader_type: "llm"` cases are SKIPPED unless `-Live` is set; under `-Live`
+    they require an `m_*` sub-agent dispatcher (not implemented in OSS — flagged
+    as skipped with reason).
+
+.PARAMETER Skill
+Single skill to run. Conflicts with -All / -Canary.
+
+.PARAMETER All
+Run every plugin/skills/<name>/evals/evals.json (except `eval/` and `self-check/`).
+
+.PARAMETER Canary
+Run only cases marked `"canary": true`.
+
+.PARAMETER Output
+Override the per-run JSON output path.
+
+.PARAMETER Baseline
+Compare against `evals/baseline.json` (default ON).
+
+.PARAMETER UpdateBaseline
+Write the current run's metrics into `evals/baseline.json` instead of comparing.
+
+.PARAMETER Live
+Allow LLM-rubric cases to attempt to dispatch (not implemented in OSS; will skip).
+
+.PARAMETER StrictExit
+Exit code 1 if any case fails (default: exit 1 only on regression vs baseline).
+
+.PARAMETER Root
+Repo root (default: two levels above this script).
+#>
+[CmdletBinding()]
+param(
+    [string]$Skill,
+    [switch]$All,
+    [switch]$Canary,
+    [string]$Output,
+    [switch]$Baseline = $true,
+    [switch]$UpdateBaseline,
+    [switch]$Live,
+    [switch]$StrictExit,
+    [string]$Root = (Resolve-Path (Join-Path $PSScriptRoot "..\..\..")).Path
+)
+
+$ErrorActionPreference = 'Stop'
+
+function Write-Info($m) { Write-Host "kushi-eval: $m" }
+function Write-Warn($m) { Write-Host "kushi-eval: $m" -ForegroundColor Yellow }
+function Write-Err($m)  { Write-Host "kushi-eval: $m" -ForegroundColor Red }
+
+# ---------- Discover skills ----------
+
+function Get-EvalFiles {
+    param([string]$Root, [string]$Skill)
+    $skillsDir = Join-Path $Root 'plugin/skills'
+    if ($Skill) {
+        $f = Join-Path $skillsDir "$Skill/evals/evals.json"
+        if (-not (Test-Path $f)) { throw "No evals.json for skill '$Skill' at $f" }
+        return ,$f
+    }
+    # Discover every skill that ships an evals/evals.json. The eval skill itself is
+    # excluded (it has no SKILL evals — it IS the runner). self-check IS included
+    # because it ships its own meta-evals.
+    return Get-ChildItem -Path $skillsDir -Directory |
+        Where-Object { $_.Name -ne 'eval' } |
+        ForEach-Object { Join-Path $_.FullName 'evals/evals.json' } |
+        Where-Object { Test-Path $_ }
+}
+
+# ---------- Lightweight JSON-Schema-ish validator (just the cases we care about) ----------
+
+function Test-EvalsShape {
+    param([Parameter(Mandatory)] $Obj, [string]$Path)
+    $problems = New-Object System.Collections.Generic.List[string]
+    if (-not $Obj.skill)  { $problems.Add("${Path}: missing 'skill'") }
+    if (-not $Obj.cases -or $Obj.cases.Count -lt 1) { $problems.Add("${Path}: needs >=1 case") }
+    foreach ($c in $Obj.cases) {
+        foreach ($req in 'id', 'name', 'input', 'grader_type', 'expected_assertions') {
+            if (-not $c.PSObject.Properties[$req]) { $problems.Add("${Path}: case missing '$req'") }
+        }
+        if ($c.grader_type -and ($c.grader_type -notin 'script', 'llm')) {
+            $problems.Add("${Path}: case '$($c.id)' has invalid grader_type '$($c.grader_type)'")
+        }
+        if ($c.expected_assertions -and $c.expected_assertions.Count -lt 1) {
+            $problems.Add("${Path}: case '$($c.id)' needs >=1 assertion")
+        }
+    }
+    return $problems
+}
+
+# ---------- Dispatch a single case ----------
+
+function Invoke-Case {
+    param(
+        [string]$Root,
+        [string]$SkillName,
+        $Case
+    )
+    $skillDir = Join-Path $Root "plugin/skills/$SkillName"
+    $probePs1 = Join-Path $skillDir 'evals/probe.ps1'
+    $probeMjs = Join-Path $skillDir 'evals/probe.mjs'
+
+    $result = [ordered]@{
+        id = $Case.id
+        name = $Case.name
+        canary = [bool]$Case.canary
+        grader_type = $Case.grader_type
+        pass = $false
+        duration_ms = 0
+        tokens_in = 0
+        tokens_out = 0
+        stdout = ''
+        stderr = ''
+        assertions = @()
+        error = $null
+    }
+
+    if ($Case.skip) {
+        $result.error = "skipped: $($Case.skip_reason)"
+        return [pscustomobject]$result
+    }
+    if ($Case.grader_type -eq 'llm' -and -not $Live) {
+        $result.error = 'skipped: llm-rubric cases require -Live'
+        return [pscustomobject]$result
+    }
+    if ($Case.grader_type -eq 'llm' -and $Live) {
+        $result.error = 'skipped: llm dispatch not implemented in OSS runner'
+        return [pscustomobject]$result
+    }
+
+    $fixturePath = $null
+    if ($Case.fixture) { $fixturePath = Join-Path $Root $Case.fixture }
+
+    $inputStr = if ($Case.input -is [string]) { $Case.input } else { ($Case.input | ConvertTo-Json -Compress -Depth 8) }
+
+    $sw = [System.Diagnostics.Stopwatch]::StartNew()
+    try {
+        if (Test-Path $probePs1) {
+            $output = & pwsh -NoLogo -NoProfile -File $probePs1 -Input $inputStr -Fixture $fixturePath -CaseId $Case.id 2>&1
+            $result.stdout = ($output | Out-String).Trim()
+        } elseif (Test-Path $probeMjs) {
+            $output = & node $probeMjs --input $inputStr --fixture ($fixturePath ?? '') --case-id $Case.id 2>&1
+            $result.stdout = ($output | Out-String).Trim()
+        } else {
+            # Built-in fixture-echo probe.
+            $readPath = $null
+            if ($Case.args -and $Case.args.read_fixture) {
+                if ($fixturePath) { $readPath = Join-Path $fixturePath $Case.args.read_fixture }
+                else { $readPath = Join-Path $Root $Case.args.read_fixture }
+            }
+            if ($readPath -and (Test-Path $readPath)) {
+                $result.stdout = Get-Content -Raw -Path $readPath
+            } else {
+                # Default: echo the input as the "answer".
+                $result.stdout = $inputStr
+            }
+        }
+    } catch {
+        $result.error = $_.Exception.Message
+    } finally {
+        $sw.Stop()
+        $result.duration_ms = [int]$sw.Elapsed.TotalMilliseconds
+    }
+
+    # Rough token estimate: 4 chars/token.
+    $result.tokens_in  = [int][math]::Ceiling($inputStr.Length / 4.0)
+    $result.tokens_out = [int][math]::Ceiling($result.stdout.Length / 4.0)
+
+    # Run assertions
+    $allPass = $true
+    foreach ($a in $Case.expected_assertions) {
+        $ares = Invoke-Assertion -Assertion $a -CaseResult $result -FixturePath $fixturePath -Root $Root
+        $result.assertions += $ares
+        if (-not $ares.pass) { $allPass = $false }
+    }
+    if ($result.error) { $allPass = $false }
+    $result.pass = $allPass
+    return [pscustomobject]$result
+}
+
+# ---------- Assertion implementations ----------
+
+function Resolve-AssertPath {
+    param([string]$P, [string]$FixturePath, [string]$Root)
+    if ([System.IO.Path]::IsPathRooted($P)) { return $P }
+    if ($FixturePath -and (Test-Path (Join-Path $FixturePath $P))) { return Join-Path $FixturePath $P }
+    return Join-Path $Root $P
+}
+
+function Invoke-Assertion {
+    param($Assertion, $CaseResult, [string]$FixturePath, [string]$Root)
+    $ar = [ordered]@{ type = $Assertion.type; pass = $false; reason = '' }
+    try {
+        switch ($Assertion.type) {
+            'file-exists' {
+                $p = Resolve-AssertPath -P $Assertion.path -FixturePath $FixturePath -Root $Root
+                $ar.pass = Test-Path $p
+                if (-not $ar.pass) { $ar.reason = "missing: $p" }
+            }
+            'file-contains' {
+                $p = Resolve-AssertPath -P $Assertion.path -FixturePath $FixturePath -Root $Root
+                if (-not (Test-Path $p)) { $ar.reason = "missing: $p" }
+                else {
+                    $txt = Get-Content -Raw -Path $p
+                    $ar.pass = $txt -and $txt.Contains([string]$Assertion.needle)
+                    if (-not $ar.pass) { $ar.reason = "needle not found in $p" }
+                }
+            }
+            'regex-match' {
+                $flags = if ($Assertion.flags) { $Assertion.flags } else { '' }
+                $opts = [System.Text.RegularExpressions.RegexOptions]::None
+                if ($flags -match 'i') { $opts = $opts -bor [System.Text.RegularExpressions.RegexOptions]::IgnoreCase }
+                if ($flags -match 'm') { $opts = $opts -bor [System.Text.RegularExpressions.RegexOptions]::Multiline }
+                $rx = [System.Text.RegularExpressions.Regex]::new($Assertion.pattern, $opts)
+                $hay = "$($CaseResult.stdout)`n$($CaseResult.stderr)"
+                $ar.pass = $rx.IsMatch($hay)
+                if (-not $ar.pass) { $ar.reason = "no match for /$($Assertion.pattern)/" }
+            }
+            'json-path-equals' {
+                $p = Resolve-AssertPath -P $Assertion.path -FixturePath $FixturePath -Root $Root
+                if (-not (Test-Path $p)) { $ar.reason = "missing: $p" }
+                else {
+                    $j = Get-Content -Raw -Path $p | ConvertFrom-Json
+                    $val = $j
+                    foreach ($seg in ($Assertion.json_path -replace '^\$\.?', '' -split '\.' | Where-Object { $_ })) {
+                        if ($null -eq $val) { break }
+                        $val = $val.$seg
+                    }
+                    if ($val -is [array]) { $val = ($val -join ',') }
+                    $ar.pass = ("$val" -eq "$($Assertion.equals)")
+                    if (-not $ar.pass) { $ar.reason = "expected '$($Assertion.equals)', got '$val'" }
+                }
+            }
+            'llm-rubric' {
+                $ar.pass = $false
+                $ar.reason = 'llm-rubric not evaluable in script runner'
+            }
+            default {
+                $ar.reason = "unknown assertion type: $($Assertion.type)"
+            }
+        }
+    } catch {
+        $ar.reason = "assertion error: $($_.Exception.Message)"
+    }
+    return [pscustomobject]$ar
+}
+
+# ---------- Main ----------
+
+if ($Skill -and ($All -or $Canary)) {
+    Write-Err "-Skill cannot be combined with -All / -Canary"; exit 2
+}
+if (-not $Skill -and -not $All -and -not $Canary) {
+    Write-Err "Pick a mode: -Skill <name> | -All | -Canary"; exit 2
+}
+
+$evalFiles = Get-EvalFiles -Root $Root -Skill $Skill
+
+if (-not $Output) {
+    $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd-HHmmss')
+    $Output = Join-Path $Root "Evidence/_evals/$stamp.json"
+}
+New-Item -ItemType Directory -Force -Path (Split-Path $Output -Parent) | Out-Null
+
+$runRecord = [ordered]@{
+    schema = 'kushi.evals.run/v1'
+    generated_at = (Get-Date).ToUniversalTime().ToString('o')
+    mode = if ($Skill) { "skill:$Skill" } elseif ($Canary) { 'canary' } else { 'all' }
+    root = $Root
+    skills = @()
+}
+$schemaProblems = New-Object System.Collections.Generic.List[string]
+$anyCaseFailed = $false
+
+foreach ($f in $evalFiles) {
+    $obj = Get-Content -Raw $f | ConvertFrom-Json
+    $problems = Test-EvalsShape -Obj $obj -Path $f
+    foreach ($p in $problems) { $schemaProblems.Add($p) }
+    if ($problems.Count -gt 0) { continue }
+
+    $cases = $obj.cases
+    if ($Canary) { $cases = $cases | Where-Object { $_.canary } }
+    $skillEntry = [ordered]@{
+        skill = $obj.skill
+        evals_file = (Resolve-Path -Relative $f)
+        cases = @()
+        pass_count = 0
+        fail_count = 0
+        skipped_count = 0
+    }
+
+    foreach ($c in $cases) {
+        Write-Info "[$($obj.skill)] $($c.id) ..."
+        $r = Invoke-Case -Root $Root -SkillName $obj.skill -Case $c
+        $skillEntry.cases += $r
+        if ($r.error -and $r.error.StartsWith('skipped')) { $skillEntry.skipped_count++ }
+        elseif ($r.pass) { $skillEntry.pass_count++ }
+        else { $skillEntry.fail_count++; $anyCaseFailed = $true }
+    }
+    $runRecord.skills += $skillEntry
+}
+
+if ($schemaProblems.Count -gt 0) {
+    Write-Err "Schema problems:"
+    foreach ($p in $schemaProblems) { Write-Err "  - $p" }
+    exit 2
+}
+
+$runRecord | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $Output -Encoding UTF8
+Write-Info "Run written: $Output"
+
+# Aggregator + baseline compare
+$aggregator = Join-Path $Root 'src/eval-aggregator.mjs'
+$baselineFile = Join-Path $Root 'evals/baseline.json'
+$benchFile = Join-Path $Root 'Evidence/_evals/benchmark.json'
+
+if (Test-Path $aggregator) {
+    $aggArgs = @($aggregator, '--run', $Output, '--bench', $benchFile)
+    if ((Test-Path $baselineFile -ErrorAction SilentlyContinue) -or $UpdateBaseline) {
+        $aggArgs += @('--baseline', $baselineFile)
+    }
+    if ($UpdateBaseline) { $aggArgs += '--update-baseline' }
+    & node @aggArgs
+    $aggExit = $LASTEXITCODE
+} else {
+    Write-Warn 'src/eval-aggregator.mjs not found — skipping aggregation'
+    $aggExit = 0
+}
+
+# Print a one-screen summary
+$total = 0; $pass = 0; $fail = 0; $skip = 0
+foreach ($s in $runRecord.skills) {
+    $total += $s.pass_count + $s.fail_count + $s.skipped_count
+    $pass += $s.pass_count
+    $fail += $s.fail_count
+    $skip += $s.skipped_count
+}
+Write-Host ""
+Write-Host "### Kushi evals — $($runRecord.mode)"
+Write-Host "| Skill | Pass | Fail | Skip |"
+Write-Host "|-------|-----:|-----:|-----:|"
+foreach ($s in $runRecord.skills) {
+    Write-Host ("| {0} | {1} | {2} | {3} |" -f $s.skill, $s.pass_count, $s.fail_count, $s.skipped_count)
+}
+Write-Host ""
+Write-Host "Total: $total · Pass: $pass · Fail: $fail · Skip: $skip"
+Write-Host "Run JSON: $Output"
+if (Test-Path $benchFile) { Write-Host "Benchmark: $benchFile" }
+
+if ($StrictExit -and $anyCaseFailed) { exit 1 }
+if ($aggExit -ne 0) { exit $aggExit }
+exit 0