kushi-agents 5.0.1 → 5.0.3

package/plugin/skills/apply-ado-update/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "apply-ado-update",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for apply-ado-update. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "apply-ado-update-smoke-1",
+      "name": "apply-ado-update produces a non-empty response",
+      "input": "synthetic apply-ado-update probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "apply-ado-update-smoke-2",
+      "name": "apply-ado-update echoes case id",
+      "input": "case-id apply-ado-update-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "apply-ado-update-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/ask-project/evals/evals.json

@@ -0,0 +1,34 @@
+{
+  "skill": "ask-project",
+  "version": "1.0.0",
+  "description": "Verifies citation format (weekly-csc) and Source-layout footer.",
+  "cases": [
+    {
+      "id": "ap-macc-citation",
+      "name": "answers MACC question with weekly-csc citation",
+      "input": "what was decided about MACC for fixture-acme?",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "args": { "read_fixture": "../outputs/ask-project.macc.txt" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "MACC" },
+        { "type": "regex-match", "pattern": "\\[source:\\s*fixture-acme/email/weekly/" },
+        { "type": "regex-match", "pattern": "Source-layout:\\s*weekly-csc" }
+      ]
+    },
+    {
+      "id": "ap-em-question",
+      "name": "answers who-is-the-EM with cited person",
+      "input": "who is the lead on fixture-acme?",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "args": { "read_fixture": "../outputs/ask-project.lead.txt" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "alice@fixture\\.local" },
+        { "type": "regex-match", "pattern": "Source-layout:\\s*weekly-csc" }
+      ]
+    }
+  ]
+}

package/plugin/skills/bootstrap-project/evals/evals.json

@@ -0,0 +1,34 @@
+{
+  "skill": "bootstrap-project",
+  "version": "1.0.0",
+  "description": "Asserts the canonical engagement folder layout from a dry-run.",
+  "cases": [
+    {
+      "id": "bp-dryrun-layout",
+      "name": "dry-run prints the canonical layout",
+      "input": "bootstrap fixture-acme --dry-run",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "args": { "read_fixture": "../outputs/bootstrap-project.dryrun.txt" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "DRY-RUN" },
+        { "type": "regex-match", "pattern": "fixture-acme/Evidence/fixture-alias/email/weekly/" },
+        { "type": "regex-match", "pattern": "fixture-acme/State/" },
+        { "type": "regex-match", "pattern": "fixture-acme/Evidence/_graph/" }
+      ]
+    },
+    {
+      "id": "bp-fixture-tree-present",
+      "name": "fixture engagement tree exists on disk",
+      "input": "verify fixture tree",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "Evidence/fixture-alias/email/weekly/2026-05-18_email-csc.md" },
+        { "type": "file-exists", "path": "State/index.md" }
+      ]
+    }
+  ]
+}

package/plugin/skills/build-state/evals/evals.json

@@ -0,0 +1,31 @@
+{
+  "skill": "build-state",
+  "version": "1.0.0",
+  "description": "Karpathy State layout — index.md + log.md + per-category folders.",
+  "cases": [
+    {
+      "id": "bs-state-index",
+      "name": "fixture State/index.md has kushi_state_page front-matter",
+      "input": "validate state fixture",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "args": { "read_fixture": "State/index.md" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "kushi_state_page:\\s*true" }
+      ]
+    },
+    {
+      "id": "bs-state-log-exists",
+      "name": "fixture State/log.md exists",
+      "input": "verify log",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "State/log.md" },
+        { "type": "file-exists", "path": "State/index.md" }
+      ]
+    }
+  ]
+}

package/plugin/skills/consolidate-evidence/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "consolidate-evidence",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for consolidate-evidence. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "consolidate-evidence-smoke-1",
+      "name": "consolidate-evidence produces a non-empty response",
+      "input": "synthetic consolidate-evidence probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "consolidate-evidence-smoke-2",
+      "name": "consolidate-evidence echoes case id",
+      "input": "case-id consolidate-evidence-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "consolidate-evidence-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/dashboard/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "dashboard",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for dashboard. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "dashboard-smoke-1",
+      "name": "dashboard produces a non-empty response",
+      "input": "synthetic dashboard probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "dashboard-smoke-2",
+      "name": "dashboard echoes case id",
+      "input": "case-id dashboard-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "dashboard-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/emit-vertex/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "emit-vertex",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for emit-vertex. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "emit-vertex-smoke-1",
+      "name": "emit-vertex produces a non-empty response",
+      "input": "synthetic emit-vertex probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "emit-vertex-smoke-2",
+      "name": "emit-vertex echoes case id",
+      "input": "case-id emit-vertex-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "emit-vertex-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/eval/SKILL.md

@@ -0,0 +1,90 @@
+---
+name: "eval"
+version: "1.0.0"
+description: "USE WHEN the user says \"run evals\", \"eval canary\", \"eval ask-project\", \"check skill regression\", \"update eval baseline\", or before tagging a release. DO NOT USE for evidence validation of a real project (use ask-project / project-status). Capability: runs per-skill evals (deterministic script graders + optional LLM-rubric graders) from each skill's evals/evals.json, aggregates pass-rate / latency / token metrics, and compares against evals/baseline.json to flag regressions. Synthetic fixtures only — never live customer data."
+---
+
+# Skill: eval
+
+The objective regression signal for every other kushi skill. Spec: <https://agentskills.io/skill-creation/evaluating-skills>. Doctrine: `instructions/skill-evals.instructions.md`.
+
+User triggers: "run evals", "eval canary", "eval <skill>", "eval all", "update eval baseline", "check skill regression".
+
+## USE WHEN
+
+- About to commit a behavioral change to any `plugin/skills/<name>/SKILL.md` or its companion script.
+- Reviewing a PR that touches skills, prompts, or shared instructions.
+- Tagging a release (full suite is implied by the release script).
+- Investigating "did this skill get worse?" after a refactor.
+
+## DO NOT USE FOR
+
+- Validating real customer evidence (use `ask-project` / `project-status`).
+- Live M365 / ADO / CRM probes (use `pull-*` directly).
+- Generating new fixtures from real engagements — that violates the privacy rule.
+
+## Gotchas
+
+1. **Canary ≠ full.** `npm run eval:canary` runs ~6 skills. Use `npm run eval:all` before tagging.
+2. **Synthetic fixtures only.** Never copy real customer evidence into `evals/fixtures/`. The runner does NOT enforce this — humans do.
+3. **`pull-*` skills run in `--cached` / `--dry-run`.** Live network calls are explicitly disabled in canary mode; the LLM-rubric subset is skipped unless `-Live` is passed.
+4. **Baseline drift is OK after intentional changes.** Refresh with `npm run eval:baseline`. NEVER auto-update baseline in CI.
+5. **`Evidence/_evals/` is gitignored.** If you need to share a run, copy the JSON manually.
+6. **Tokens are estimates.** When a grader can't measure tokens (e.g. a pwsh-only run), it records `0` and the aggregator excludes those from the mean.
+
+## Step checklist
+
+- [ ] Pick mode: `-Skill <name>` (one skill) · `-Canary` (fast subset) · `-All` (full suite).
+- [ ] Confirm `evals/baseline.json` exists; if missing, run with `-UpdateBaseline` first.
+- [ ] Run: `pwsh plugin/skills/eval/run-evals.ps1 -Canary` (or other mode).
+- [ ] Inspect the output JSON under `Evidence/_evals/` and the `benchmark.json` summary.
+- [ ] If regressions flagged: re-run the specific skill with `-Skill <name>` for detail.
+- [ ] If intentional change: bump the skill version, then `npm run eval:baseline`.
+
+## Validation loop
+
+After running, verify:
+
+- Output JSON exists at the path printed by the runner.
+- Every case has `pass: true|false` AND `duration_ms` AND a per-assertion breakdown.
+- `benchmark.json` has `summary.regressions` array (empty if clean).
+- Exit code: `0` = all green vs baseline; `1` = at least one regression.
+
+If a case errors out before any assertion runs (e.g. fixture missing), it counts as `pass: false` with `error: "<message>"` so totals stay honest.
+
+## What the runner does (`run-evals.ps1`)
+
+1. Discovers `plugin/skills/<name>/evals/evals.json` for the requested skills.
+2. Validates each file against `plugin/skills/eval/evals.schema.json`.
+3. For each case:
+   - Resolves the fixture path (`fixture` field, relative to repo root).
+   - Dispatches per `grader_type`:
+     - `script` → invokes the skill's known executable (`run.ps1`, a `*.mjs` probe, or a deterministic shim under `evals/probe.*`) with the case input.
+     - `llm` → only when `-Live` is set; otherwise marked `skipped`.
+   - Captures `stdout`, `stderr`, `duration_ms`.
+   - Runs every assertion in `expected_assertions`; collects per-assertion pass/fail.
+   - Case `pass` = all assertions pass AND no error.
+4. Writes per-run JSON to `Evidence/_evals/<utc-timestamp>.json` (or `-Output` override).
+5. Calls `node src/eval-aggregator.mjs` to compute `benchmark.json` and compare against `evals/baseline.json`.
+6. Prints a one-screen summary; exits 0 on clean, 1 on regression.
+
+## Arguments
+
+| Flag | Purpose |
+|---|---|
+| `-Skill <name>` | Run only one skill's evals. |
+| `-All` | Run every `plugin/skills/<name>/evals/evals.json`. |
+| `-Canary` | Run only cases marked `"canary": true`. |
+| `-Output <path>` | Override per-run JSON path. Default `Evidence/_evals/<ts>.json`. |
+| `-Baseline` | Compare against `evals/baseline.json` (default ON). |
+| `-UpdateBaseline` | Write current run's metrics into `evals/baseline.json`. |
+| `-Live` | Allow LLM-rubric cases (requires `m_*` tools / sub-agent). |
+| `-StrictExit` | Exit 1 on any case failure (CI mode). |
+
+## References
+
+- `plugin/instructions/skill-evals.instructions.md` (doctrine)
+- `plugin/instructions/agentskills-compliance.instructions.md` (size + section caps for SKILL.md)
+- `src/eval-aggregator.mjs` (mean/stddev + regression detection)
+- `evals/baseline.json` (committed baseline)
+- <https://agentskills.io/skill-creation/evaluating-skills>

package/plugin/skills/eval/evals.schema.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://gim-home.github.io/kushi/schemas/evals.schema.json",
+  "title": "Kushi per-skill evals file",
+  "description": "Schema for plugin/skills/<name>/evals/evals.json. Validated by self-check D33.evals-schema.",
+  "type": "object",
+  "required": ["skill", "cases"],
+  "additionalProperties": false,
+  "properties": {
+    "skill": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Skill name (must match the parent directory and SKILL.md frontmatter name)."
+    },
+    "version": { "type": "string" },
+    "description": { "type": "string" },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/case" }
+    }
+  },
+  "definitions": {
+    "case": {
+      "type": "object",
+      "required": ["id", "name", "input", "expected_assertions", "grader_type"],
+      "additionalProperties": false,
+      "properties": {
+        "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" },
+        "name": { "type": "string", "minLength": 1 },
+        "input": {
+          "anyOf": [{ "type": "string" }, { "type": "object" }]
+        },
+        "fixture": { "type": "string" },
+        "canary": { "type": "boolean", "default": false },
+        "skip": { "type": "boolean", "default": false },
+        "skip_reason": { "type": "string" },
+        "timeout_ms": { "type": "integer", "minimum": 100 },
+        "args": { "type": "object" },
+        "grader_type": { "enum": ["script", "llm"] },
+        "expected_assertions": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/definitions/assertion" }
+        }
+      }
+    },
+    "assertion": {
+      "type": "object",
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "enum": [
+            "file-exists",
+            "file-contains",
+            "json-path-equals",
+            "regex-match",
+            "llm-rubric"
+          ]
+        },
+        "path": { "type": "string" },
+        "needle": { "type": "string" },
+        "pattern": { "type": "string" },
+        "flags": { "type": "string" },
+        "json_path": { "type": "string" },
+        "equals": {},
+        "rubric": { "type": "string" },
+        "min_score": { "type": "number" }
+      },
+      "additionalProperties": false
+    }
+  }
+}