kushi-agents 5.0.2 → 5.0.3

package/plugin/skills/eval/run-evals.ps1

@@ -0,0 +1,372 @@
+<#
+.SYNOPSIS
+Kushi skill evals runner (v5.0.3+).
+
+.DESCRIPTION
+Discovers per-skill evals at `plugin/skills/<name>/evals/evals.json`,
+validates against `plugin/skills/eval/evals.schema.json`, dispatches each case,
+runs the assertions, and writes a per-run JSON to `Evidence/_evals/<ts>.json`.
+
+Per the doctrine in `plugin/instructions/skill-evals.instructions.md`, the
+runner NEVER makes live M365 / ADO / CRM calls. Skills that genuinely require
+live calls (`pull-*`) ship cached fixtures.
+
+Dispatch:
+  - `grader_type: "script"` cases are dispatched to:
+      1. `plugin/skills/<skill>/evals/probe.ps1`  (preferred, per-skill probe)
+      2. `plugin/skills/<skill>/evals/probe.mjs`  (Node probe)
+      3. The built-in "fixture-echo" probe, which reads `case.args.read_fixture`
+         and emits its content. Lets simple read-only assertions work with no
+         per-skill code at all.
+  - `grader_type: "llm"` cases are SKIPPED unless `-Live` is set; under `-Live`
+    they require an `m_*` sub-agent dispatcher (not implemented in OSS — flagged
+    as skipped with reason).
+
+.PARAMETER Skill
+Single skill to run. Conflicts with -All / -Canary.
+
+.PARAMETER All
+Run every plugin/skills/<name>/evals/evals.json (except `eval/` and `self-check/`).
+
+.PARAMETER Canary
+Run only cases marked `"canary": true`.
+
+.PARAMETER Output
+Override the per-run JSON output path.
+
+.PARAMETER Baseline
+Compare against `evals/baseline.json` (default ON).
+
+.PARAMETER UpdateBaseline
+Write the current run's metrics into `evals/baseline.json` instead of comparing.
+
+.PARAMETER Live
+Allow LLM-rubric cases to attempt to dispatch (not implemented in OSS; will skip).
+
+.PARAMETER StrictExit
+Exit code 1 if any case fails (default: exit 1 only on regression vs baseline).
+
+.PARAMETER Root
+Repo root (default: two levels above this script).
+#>
+[CmdletBinding()]
+param(
+    [string]$Skill,
+    [switch]$All,
+    [switch]$Canary,
+    [string]$Output,
+    [switch]$Baseline = $true,
+    [switch]$UpdateBaseline,
+    [switch]$Live,
+    [switch]$StrictExit,
+    [string]$Root = (Resolve-Path (Join-Path $PSScriptRoot "..\..\..")).Path
+)
+
+$ErrorActionPreference = 'Stop'
+
+function Write-Info($m) { Write-Host "kushi-eval: $m" }
+function Write-Warn($m) { Write-Host "kushi-eval: $m" -ForegroundColor Yellow }
+function Write-Err($m)  { Write-Host "kushi-eval: $m" -ForegroundColor Red }
+
+# ---------- Discover skills ----------
+
+function Get-EvalFiles {
+    param([string]$Root, [string]$Skill)
+    $skillsDir = Join-Path $Root 'plugin/skills'
+    if ($Skill) {
+        $f = Join-Path $skillsDir "$Skill/evals/evals.json"
+        if (-not (Test-Path $f)) { throw "No evals.json for skill '$Skill' at $f" }
+        return ,$f
+    }
+    # Discover every skill that ships an evals/evals.json. The eval skill itself is
+    # excluded (it has no SKILL evals — it IS the runner). self-check IS included
+    # because it ships its own meta-evals.
+    return Get-ChildItem -Path $skillsDir -Directory |
+        Where-Object { $_.Name -ne 'eval' } |
+        ForEach-Object { Join-Path $_.FullName 'evals/evals.json' } |
+        Where-Object { Test-Path $_ }
+}
+
+# ---------- Lightweight JSON-Schema-ish validator (just the cases we care about) ----------
+
+function Test-EvalsShape {
+    param([Parameter(Mandatory)] $Obj, [string]$Path)
+    $problems = New-Object System.Collections.Generic.List[string]
+    if (-not $Obj.skill)  { $problems.Add("${Path}: missing 'skill'") }
+    if (-not $Obj.cases -or $Obj.cases.Count -lt 1) { $problems.Add("${Path}: needs >=1 case") }
+    foreach ($c in $Obj.cases) {
+        foreach ($req in 'id', 'name', 'input', 'grader_type', 'expected_assertions') {
+            if (-not $c.PSObject.Properties[$req]) { $problems.Add("${Path}: case missing '$req'") }
+        }
+        if ($c.grader_type -and ($c.grader_type -notin 'script', 'llm')) {
+            $problems.Add("${Path}: case '$($c.id)' has invalid grader_type '$($c.grader_type)'")
+        }
+        if ($c.expected_assertions -and $c.expected_assertions.Count -lt 1) {
+            $problems.Add("${Path}: case '$($c.id)' needs >=1 assertion")
+        }
+    }
+    return $problems
+}
+
+# ---------- Dispatch a single case ----------
+
+function Invoke-Case {
+    param(
+        [string]$Root,
+        [string]$SkillName,
+        $Case
+    )
+    $skillDir = Join-Path $Root "plugin/skills/$SkillName"
+    $probePs1 = Join-Path $skillDir 'evals/probe.ps1'
+    $probeMjs = Join-Path $skillDir 'evals/probe.mjs'
+
+    $result = [ordered]@{
+        id = $Case.id
+        name = $Case.name
+        canary = [bool]$Case.canary
+        grader_type = $Case.grader_type
+        pass = $false
+        duration_ms = 0
+        tokens_in = 0
+        tokens_out = 0
+        stdout = ''
+        stderr = ''
+        assertions = @()
+        error = $null
+    }
+
+    if ($Case.skip) {
+        $result.error = "skipped: $($Case.skip_reason)"
+        return [pscustomobject]$result
+    }
+    if ($Case.grader_type -eq 'llm' -and -not $Live) {
+        $result.error = 'skipped: llm-rubric cases require -Live'
+        return [pscustomobject]$result
+    }
+    if ($Case.grader_type -eq 'llm' -and $Live) {
+        $result.error = 'skipped: llm dispatch not implemented in OSS runner'
+        return [pscustomobject]$result
+    }
+
+    $fixturePath = $null
+    if ($Case.fixture) { $fixturePath = Join-Path $Root $Case.fixture }
+
+    $inputStr = if ($Case.input -is [string]) { $Case.input } else { ($Case.input | ConvertTo-Json -Compress -Depth 8) }
+
+    $sw = [System.Diagnostics.Stopwatch]::StartNew()
+    try {
+        if (Test-Path $probePs1) {
+            $output = & pwsh -NoLogo -NoProfile -File $probePs1 -Input $inputStr -Fixture $fixturePath -CaseId $Case.id 2>&1
+            $result.stdout = ($output | Out-String).Trim()
+        } elseif (Test-Path $probeMjs) {
+            $output = & node $probeMjs --input $inputStr --fixture ($fixturePath ?? '') --case-id $Case.id 2>&1
+            $result.stdout = ($output | Out-String).Trim()
+        } else {
+            # Built-in fixture-echo probe.
+            $readPath = $null
+            if ($Case.args -and $Case.args.read_fixture) {
+                if ($fixturePath) { $readPath = Join-Path $fixturePath $Case.args.read_fixture }
+                else { $readPath = Join-Path $Root $Case.args.read_fixture }
+            }
+            if ($readPath -and (Test-Path $readPath)) {
+                $result.stdout = Get-Content -Raw -Path $readPath
+            } else {
+                # Default: echo the input as the "answer".
+                $result.stdout = $inputStr
+            }
+        }
+    } catch {
+        $result.error = $_.Exception.Message
+    } finally {
+        $sw.Stop()
+        $result.duration_ms = [int]$sw.Elapsed.TotalMilliseconds
+    }
+
+    # Rough token estimate: 4 chars/token.
+    $result.tokens_in  = [int][math]::Ceiling($inputStr.Length / 4.0)
+    $result.tokens_out = [int][math]::Ceiling($result.stdout.Length / 4.0)
+
+    # Run assertions
+    $allPass = $true
+    foreach ($a in $Case.expected_assertions) {
+        $ares = Invoke-Assertion -Assertion $a -CaseResult $result -FixturePath $fixturePath -Root $Root
+        $result.assertions += $ares
+        if (-not $ares.pass) { $allPass = $false }
+    }
+    if ($result.error) { $allPass = $false }
+    $result.pass = $allPass
+    return [pscustomobject]$result
+}
+
+# ---------- Assertion implementations ----------
+
+function Resolve-AssertPath {
+    param([string]$P, [string]$FixturePath, [string]$Root)
+    if ([System.IO.Path]::IsPathRooted($P)) { return $P }
+    if ($FixturePath -and (Test-Path (Join-Path $FixturePath $P))) { return Join-Path $FixturePath $P }
+    return Join-Path $Root $P
+}
+
+function Invoke-Assertion {
+    param($Assertion, $CaseResult, [string]$FixturePath, [string]$Root)
+    $ar = [ordered]@{ type = $Assertion.type; pass = $false; reason = '' }
+    try {
+        switch ($Assertion.type) {
+            'file-exists' {
+                $p = Resolve-AssertPath -P $Assertion.path -FixturePath $FixturePath -Root $Root
+                $ar.pass = Test-Path $p
+                if (-not $ar.pass) { $ar.reason = "missing: $p" }
+            }
+            'file-contains' {
+                $p = Resolve-AssertPath -P $Assertion.path -FixturePath $FixturePath -Root $Root
+                if (-not (Test-Path $p)) { $ar.reason = "missing: $p" }
+                else {
+                    $txt = Get-Content -Raw -Path $p
+                    $ar.pass = $txt -and $txt.Contains([string]$Assertion.needle)
+                    if (-not $ar.pass) { $ar.reason = "needle not found in $p" }
+                }
+            }
+            'regex-match' {
+                $flags = if ($Assertion.flags) { $Assertion.flags } else { '' }
+                $opts = [System.Text.RegularExpressions.RegexOptions]::None
+                if ($flags -match 'i') { $opts = $opts -bor [System.Text.RegularExpressions.RegexOptions]::IgnoreCase }
+                if ($flags -match 'm') { $opts = $opts -bor [System.Text.RegularExpressions.RegexOptions]::Multiline }
+                $rx = [System.Text.RegularExpressions.Regex]::new($Assertion.pattern, $opts)
+                $hay = "$($CaseResult.stdout)`n$($CaseResult.stderr)"
+                $ar.pass = $rx.IsMatch($hay)
+                if (-not $ar.pass) { $ar.reason = "no match for /$($Assertion.pattern)/" }
+            }
+            'json-path-equals' {
+                $p = Resolve-AssertPath -P $Assertion.path -FixturePath $FixturePath -Root $Root
+                if (-not (Test-Path $p)) { $ar.reason = "missing: $p" }
+                else {
+                    $j = Get-Content -Raw -Path $p | ConvertFrom-Json
+                    $val = $j
+                    foreach ($seg in ($Assertion.json_path -replace '^\$\.?', '' -split '\.' | Where-Object { $_ })) {
+                        if ($null -eq $val) { break }
+                        $val = $val.$seg
+                    }
+                    if ($val -is [array]) { $val = ($val -join ',') }
+                    $ar.pass = ("$val" -eq "$($Assertion.equals)")
+                    if (-not $ar.pass) { $ar.reason = "expected '$($Assertion.equals)', got '$val'" }
+                }
+            }
+            'llm-rubric' {
+                $ar.pass = $false
+                $ar.reason = 'llm-rubric not evaluable in script runner'
+            }
+            default {
+                $ar.reason = "unknown assertion type: $($Assertion.type)"
+            }
+        }
+    } catch {
+        $ar.reason = "assertion error: $($_.Exception.Message)"
+    }
+    return [pscustomobject]$ar
+}
+
+# ---------- Main ----------
+
+if ($Skill -and ($All -or $Canary)) {
+    Write-Err "-Skill cannot be combined with -All / -Canary"; exit 2
+}
+if (-not $Skill -and -not $All -and -not $Canary) {
+    Write-Err "Pick a mode: -Skill <name> | -All | -Canary"; exit 2
+}
+
+$evalFiles = Get-EvalFiles -Root $Root -Skill $Skill
+
+if (-not $Output) {
+    $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd-HHmmss')
+    $Output = Join-Path $Root "Evidence/_evals/$stamp.json"
+}
+New-Item -ItemType Directory -Force -Path (Split-Path $Output -Parent) | Out-Null
+
+$runRecord = [ordered]@{
+    schema = 'kushi.evals.run/v1'
+    generated_at = (Get-Date).ToUniversalTime().ToString('o')
+    mode = if ($Skill) { "skill:$Skill" } elseif ($Canary) { 'canary' } else { 'all' }
+    root = $Root
+    skills = @()
+}
+$schemaProblems = New-Object System.Collections.Generic.List[string]
+$anyCaseFailed = $false
+
+foreach ($f in $evalFiles) {
+    $obj = Get-Content -Raw $f | ConvertFrom-Json
+    $problems = Test-EvalsShape -Obj $obj -Path $f
+    foreach ($p in $problems) { $schemaProblems.Add($p) }
+    if ($problems.Count -gt 0) { continue }
+
+    $cases = $obj.cases
+    if ($Canary) { $cases = $cases | Where-Object { $_.canary } }
+    $skillEntry = [ordered]@{
+        skill = $obj.skill
+        evals_file = (Resolve-Path -Relative $f)
+        cases = @()
+        pass_count = 0
+        fail_count = 0
+        skipped_count = 0
+    }
+
+    foreach ($c in $cases) {
+        Write-Info "[$($obj.skill)] $($c.id) ..."
+        $r = Invoke-Case -Root $Root -SkillName $obj.skill -Case $c
+        $skillEntry.cases += $r
+        if ($r.error -and $r.error.StartsWith('skipped')) { $skillEntry.skipped_count++ }
+        elseif ($r.pass) { $skillEntry.pass_count++ }
+        else { $skillEntry.fail_count++; $anyCaseFailed = $true }
+    }
+    $runRecord.skills += $skillEntry
+}
+
+if ($schemaProblems.Count -gt 0) {
+    Write-Err "Schema problems:"
+    foreach ($p in $schemaProblems) { Write-Err "  - $p" }
+    exit 2
+}
+
+$runRecord | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $Output -Encoding UTF8
+Write-Info "Run written: $Output"
+
+# Aggregator + baseline compare
+$aggregator = Join-Path $Root 'src/eval-aggregator.mjs'
+$baselineFile = Join-Path $Root 'evals/baseline.json'
+$benchFile = Join-Path $Root 'Evidence/_evals/benchmark.json'
+
+if (Test-Path $aggregator) {
+    $aggArgs = @($aggregator, '--run', $Output, '--bench', $benchFile)
+    if ((Test-Path $baselineFile -ErrorAction SilentlyContinue) -or $UpdateBaseline) {
+        $aggArgs += @('--baseline', $baselineFile)
+    }
+    if ($UpdateBaseline) { $aggArgs += '--update-baseline' }
+    & node @aggArgs
+    $aggExit = $LASTEXITCODE
+} else {
+    Write-Warn 'src/eval-aggregator.mjs not found — skipping aggregation'
+    $aggExit = 0
+}
+
+# Print a one-screen summary
+$total = 0; $pass = 0; $fail = 0; $skip = 0
+foreach ($s in $runRecord.skills) {
+    $total += $s.pass_count + $s.fail_count + $s.skipped_count
+    $pass += $s.pass_count
+    $fail += $s.fail_count
+    $skip += $s.skipped_count
+}
+Write-Host ""
+Write-Host "### Kushi evals — $($runRecord.mode)"
+Write-Host "| Skill | Pass | Fail | Skip |"
+Write-Host "|-------|-----:|-----:|-----:|"
+foreach ($s in $runRecord.skills) {
+    Write-Host ("| {0} | {1} | {2} | {3} |" -f $s.skill, $s.pass_count, $s.fail_count, $s.skipped_count)
+}
+Write-Host ""
+Write-Host "Total: $total · Pass: $pass · Fail: $fail · Skip: $skip"
+Write-Host "Run JSON: $Output"
+if (Test-Path $benchFile) { Write-Host "Benchmark: $benchFile" }
+
+if ($StrictExit -and $anyCaseFailed) { exit 1 }
+if ($aggExit -ne 0) { exit $aggExit }
+exit 0

package/plugin/skills/fde-intake/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "fde-intake",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for fde-intake. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "fde-intake-smoke-1",
+      "name": "fde-intake produces a non-empty response",
+      "input": "synthetic fde-intake probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "fde-intake-smoke-2",
+      "name": "fde-intake echoes case id",
+      "input": "case-id fde-intake-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fde-intake-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/fde-report/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "fde-report",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for fde-report. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "fde-report-smoke-1",
+      "name": "fde-report produces a non-empty response",
+      "input": "synthetic fde-report probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "fde-report-smoke-2",
+      "name": "fde-report echoes case id",
+      "input": "case-id fde-report-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fde-report-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/fde-triage/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "fde-triage",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for fde-triage. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "fde-triage-smoke-1",
+      "name": "fde-triage produces a non-empty response",
+      "input": "synthetic fde-triage probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "fde-triage-smoke-2",
+      "name": "fde-triage echoes case id",
+      "input": "case-id fde-triage-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fde-triage-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/intro/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "intro",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for intro. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "intro-smoke-1",
+      "name": "intro produces a non-empty response",
+      "input": "synthetic intro probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "intro-smoke-2",
+      "name": "intro echoes case id",
+      "input": "case-id intro-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "intro-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/link-entities/evals/evals.json

@@ -0,0 +1,31 @@
+{
+  "skill": "link-entities",
+  "version": "1.0.0",
+  "description": "Deterministic entity-graph output — closed edge taxonomy + nodes/edges schema.",
+  "cases": [
+    {
+      "id": "le-graph-schema",
+      "name": "fixture project-graph.json has v1 schema + nodes + edges",
+      "input": "validate graph schema",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "Evidence/_graph/project-graph.json" },
+        { "type": "json-path-equals", "path": "Evidence/_graph/project-graph.json", "json_path": "$.schema", "equals": "kushi.project-graph/v1" }
+      ]
+    },
+    {
+      "id": "le-closed-taxonomy",
+      "name": "every edge uses a kind from the closed taxonomy",
+      "input": "validate edges",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "args": { "read_fixture": "Evidence/_graph/project-graph.json" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "\"kind\":\\s*\"(references|decides|action-item-tracks|discusses|produced-by|follow-up-of|same-thread|participant-of)\"" }
+      ]
+    }
+  ]
+}

package/plugin/skills/project-status/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "project-status",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for project-status. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "project-status-smoke-1",
+      "name": "project-status produces a non-empty response",
+      "input": "synthetic project-status probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "project-status-smoke-2",
+      "name": "project-status echoes case id",
+      "input": "case-id project-status-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "project-status-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/propose-ado-update/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "propose-ado-update",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for propose-ado-update. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "propose-ado-update-smoke-1",
+      "name": "propose-ado-update produces a non-empty response",
+      "input": "synthetic propose-ado-update probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "propose-ado-update-smoke-2",
+      "name": "propose-ado-update echoes case id",
+      "input": "case-id propose-ado-update-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "propose-ado-update-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-ado/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-ado",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-ado. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-ado-cached-1",
+      "name": "pull-ado cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-ado-rubric-1",
+      "name": "pull-ado output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-ado pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-ado response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-crm/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-crm",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-crm. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-crm-cached-1",
+      "name": "pull-crm cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-crm-rubric-1",
+      "name": "pull-crm output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-crm pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-crm response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}