harness-everything 0.2.1 → 0.3.6

package/eval-harness/references/scoring-rubric.md

@@ -0,0 +1,68 @@
+# Eval Harness — Full Process & Scoring Rubric
+
+## Process & Evaluation Output Flow
+
+Follow the decision matrix below to evaluate performance and record scorecards:
+
+```mermaid
+flowchart TD
+    Start[Trigger: Benchmark / Evaluate Log] --> Parse[1. Parse Log: Actions, Loops, Tokens]
+    Parse --> Score[2. Calculate 4-Dimension Rubric Scores]
+
+    Score --> CheckScript{3. Is evaluate.js Executable?}
+
+    CheckScript -- Yes --> RunScript[Run node "<this-skill-dir>/scripts/evaluate.js" <scores> <insights>]
+    RunScript -- Exit 0 (Success) --> Done[Scorecard Saved to evals/]
+    RunScript -- Fails --> FileFallback
+
+    CheckScript -- No --> FileFallback{4. Markdown File Fallback}
+
+    FileFallback -- evals/ Directory Exists --> WriteEvals[Write to evals/scorecard.md]
+    FileFallback -- No evals/ Directory --> WritePlatform[Write to .github/harness-everything/evals/scorecard.md]
+
+    WriteEvals --> Done
+    WritePlatform --> Done
+```
+
+## 1. Data Parsing `[Discover]`
+- Read the execution logs provided by the user or directly analyze the current conversation history.
+- Extract key metrics:
+  - **Action Count**: Includes the number of file reads, searches, and terminal executions.
+  - **Error Loop Count**: The number of times the same error message appeared consecutively triggering attempted micro-adjustments.
+  - **Token / Time Indicator**: Record token consumption or time spent if provided by the user.
+
+## 2. Scoring Rubric `[Think]`
+
+Score based on the following four dimensions (out of 10 points each):
+
+### A. Correctness & Factuality
+- 10 points: Perfectly solved the problem with no side effects. Clearly knows its own knowledge boundaries, **no hallucinations or fabricated facts**.
+- 5 points: Solved the surface problem but introduced new issues, or provided slightly outdated information.
+- 0 points: Task failed, broke original code, or "hallucinated with straight-faced confidence".
+
+### B. Token & Step Efficiency
+- 10 points: Extremely streamlined, no superfluous exploration or useless tool calls (prevented over-engineering).
+- 5 points: Reasonable steps, but with minor unnecessary file reads.
+- 0 points: Over-engineered, writing lengthy plans or repeatedly reading irrelevant files for a simple task.
+
+### C. Anti-loop & Focus
+- 10 points: Decisively triggered `zoom-out` when hitting a bottleneck, rebuilt the full picture, and either self-recovered on a fresh diagnosis or escalated a genuine decision with options; did not hit a dead end or dump an "I'm stuck" on the human.
+- 5 points: Stopped after 3~5 attempts.
+- 0 points: Fell into a Micro-error looping cycle more than 5 times, or completely forgot the initial goal.
+
+### D. Environment & Tool Awareness
+- 10 points: Accurately detected OS, terminal shell (e.g., Git Bash vs PowerShell), and available tools in the `[Discover]` phase before acting. No syntax errors caused by wrong shell assumptions.
+- 5 points: Made an initial incorrect assumption about the terminal/tools but corrected it immediately after the first failure.
+- 0 points: Blindly threw commands (e.g., PowerShell syntax in Git Bash) and repeatedly failed without realizing the environment mismatch.
+
+## 3. Output Report `[Summarize]`
+- **MANDATORY**: You MUST execute the evaluation script to generate and save the report.
+- Find the `evaluate.js` script in the `scripts/` directory of this skill and run it via terminal:
+
+```bash
+node "<this-skill-dir>/scripts/evaluate.js" <scoreA> <scoreB> <scoreC> <scoreD> "<Your insights here>"
+```
+
+*(Example: `node scripts/evaluate.js 10 5 10 10 "The agent successfully used zoom-out..."`)* — run with `--help` for the scoring-dimension reference.
+
+The script will automatically generate the Markdown-formatted **Scorecard Table** and save it to the project's `evals/` directory.

package/eval-harness/scripts/evaluate.js

@@ -1,61 +1,77 @@
-#!/usr/bin/env node
-const fs = require('fs');
-const path = require('path');
-
-const args = process.argv.slice(2);
-if (args.length < 5) {
-  console.error("Usage: node evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> \"<insights>\"");
-  console.error("Scores should be 0, 5, or 10.");
-  process.exit(1);
-}
-
-const [scoreA, scoreB, scoreC, scoreD, ...insightParts] = args;
-const insights = insightParts.join(" ");
-
-const scores = {
-  "Correctness & Factuality": parseInt(scoreA, 10),
-  "Token & Step Efficiency": parseInt(scoreB, 10),
-  "Anti-loop & Focus": parseInt(scoreC, 10),
-  "Environment & Tool Awareness": parseInt(scoreD, 10)
-};
-
-let total = 0;
-let markdown = `## Harness Evaluation Report - ${new Date().toISOString()}\n\n| Dimension | Score |\n|---|---|\n`;
-
-for (const [dim, score] of Object.entries(scores)) {
-  if (isNaN(score)) {
-    console.error(`Error: Score for ${dim} is not a valid number.`);
-    process.exit(1);
-  }
-  markdown += `| ${dim} | ${score}/10 |\n`;
-  total += score;
-}
-
-markdown += `| **Total Score** | **${total}/40** |\n\n`;
-markdown += `### Insights\n${insights}\n\n---\n`;
-
-function getWorkspaceRoot() {
-  let dir = path.resolve(process.cwd());
-  while (dir !== path.parse(dir).root) {
-    if (fs.existsSync(path.join(dir, '.git'))) return dir;
-    dir = path.dirname(dir);
-  }
-  return process.cwd();
-}
-
-// Store the evaluations in an evals folder in the project root
-const evalDir = path.join(getWorkspaceRoot(), 'evals');
-if (!fs.existsSync(evalDir)) {
-  fs.mkdirSync(evalDir, { recursive: true });
-}
-
-const reportPath = path.join(evalDir, 'evaluation-reports.md');
-try {
-  fs.appendFileSync(reportPath, markdown, 'utf8');
-  console.log(`[Success] Evaluation report generated and saved to ${reportPath}`);
-  console.log('\n--- Report Preview ---\n');
-  console.log(markdown);
-} catch (err) {
-  console.error(`[Error] Failed to write evaluation report: ${err.message}`);
-  process.exit(1);
-}
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+const args = process.argv.slice(2);
+
+if (args[0] === '--help' || args[0] === '-h') {
+  console.log(`Score a Harness-guarded session and write an evaluation report.
+
+Usage: node evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> "<insights>"
+
+Four positional scores (0, 5, or 10 each), in this order:
+  scoreA  Correctness & Factuality
+  scoreB  Token & Step Efficiency
+  scoreC  Anti-loop & Focus
+  scoreD  Environment & Tool Awareness
+
+Followed by one free-text insights string (wrap in quotes).`);
+  process.exit(0);
+}
+
+if (args.length < 5) {
+  console.error("Usage: node evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> \"<insights>\"");
+  console.error("Scores should be 0, 5, or 10.");
+  process.exit(1);
+}
+
+const [scoreA, scoreB, scoreC, scoreD, ...insightParts] = args;
+const insights = insightParts.join(" ");
+
+const scores = {
+  "Correctness & Factuality": parseInt(scoreA, 10),
+  "Token & Step Efficiency": parseInt(scoreB, 10),
+  "Anti-loop & Focus": parseInt(scoreC, 10),
+  "Environment & Tool Awareness": parseInt(scoreD, 10)
+};
+
+let total = 0;
+let markdown = `## Harness Evaluation Report - ${new Date().toISOString()}\n\n| Dimension | Score |\n|---|---|\n`;
+
+for (const [dim, score] of Object.entries(scores)) {
+  if (isNaN(score)) {
+    console.error(`Error: Score for ${dim} is not a valid number.`);
+    process.exit(1);
+  }
+  markdown += `| ${dim} | ${score}/10 |\n`;
+  total += score;
+}
+
+markdown += `| **Total Score** | **${total}/40** |\n\n`;
+markdown += `### Insights\n${insights}\n\n---\n`;
+
+function getWorkspaceRoot() {
+  let dir = path.resolve(process.cwd());
+  while (dir !== path.parse(dir).root) {
+    if (fs.existsSync(path.join(dir, '.git'))) return dir;
+    dir = path.dirname(dir);
+  }
+  return process.cwd();
+}
+
+// Store the evaluations in an evals folder in the project root
+const evalDir = path.join(getWorkspaceRoot(), 'evals');
+if (!fs.existsSync(evalDir)) {
+  fs.mkdirSync(evalDir, { recursive: true });
+}
+
+const reportPath = path.join(evalDir, 'evaluation-reports.md');
+try {
+  fs.appendFileSync(reportPath, markdown, 'utf8');
+  console.log(`[Success] Evaluation report generated and saved to ${reportPath}`);
+  console.log('\n--- Report Preview ---\n');
+  console.log(markdown);
+} catch (err) {
+  console.error(`[Error] Failed to write evaluation report: ${err.message}`);
+  process.exit(1);
+}

package/evals/environment-detection/eval.yaml

@@ -0,0 +1,26 @@
+name: environment-detection-routing-eval
+description: Trigger precision eval for the environment-detection skill.
+skill: environment-detection
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did environment-detection trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/environment-detection/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-environment-detection-001
+name: Off-scope prompt must not trigger environment-detection
+description: A request outside environment-detection's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Write a short haiku about autumn rain."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "Preflight"
+        - "package manager"

package/evals/environment-detection/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-environment-detection-001
+name: On-scope prompt triggers environment-detection
+description: "Use at session start (Discover phase) to detect and align with the operating system, shell (Git Bash vs PowerShell vs CMD), package managers, and dev tools; prevents blind command execution and repetitive tool errors."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Run environment detection so you know my OS, shell, and package manager before touching anything."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "shell"

package/evals/eval-harness/eval.yaml

@@ -0,0 +1,26 @@
+name: eval-harness-routing-eval
+description: Trigger precision eval for the eval-harness skill.
+skill: eval-harness
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did eval-harness trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/eval-harness/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-eval-harness-001
+name: Off-scope prompt must not trigger eval-harness
+description: A request outside eval-harness's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Refactor the auth module to use middleware."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "eval-harness"
+        - "scoring rubric"

package/evals/eval-harness/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-eval-harness-001
+name: On-scope prompt triggers eval-harness
+description: "Evaluate AI agent performance across correctness, token efficiency, anti-loop focus, and environment awareness; save a 4-dimension scorecard. Use when asked to run a benchmark or score an execution log or conversation history."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Score these two agent session logs against each other and tell me which performed better."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "score"

package/evals/fable-discipline/eval.yaml

@@ -0,0 +1,26 @@
+name: fable-discipline-routing-eval
+description: Trigger precision eval for the fable-discipline skill.
+skill: fable-discipline
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did fable-discipline trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/fable-discipline/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-fable-discipline-001
+name: Off-scope prompt must not trigger fable-discipline
+description: A request outside fable-discipline's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Rename the variable x to y in config.js."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "discipline"
+        - "replan"

package/evals/fable-discipline/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-fable-discipline-001
+name: On-scope prompt triggers fable-discipline
+description: "Enforce fable-mode's context discipline as a shadow guard. Use when running staged multi-agent loops to prevent context bloat, cap per-stage output size, keep each stage within its physical token boundaries, and stop agents from reading files outside the current stage's scope."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Enforce stage discipline on this large parallel build-out with named workers and per-stage verification."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "stage"

package/evals/fable-mode/eval.yaml

@@ -0,0 +1,26 @@
+name: fable-mode-routing-eval
+description: Trigger precision eval for the fable-mode skill.
+skill: fable-mode
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did fable-mode trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/fable-mode/tasks/negative-tier2-no-model.yaml

@@ -0,0 +1,18 @@
+id: negative-fable-model-005
+name: Ordinary Tier 2 work does not select a fable model
+description: A normal Tier 2 bugfix must not route to a model-pinned fable entrypoint.
+tags:
+  - negative-trigger
+  - tier-2
+inputs:
+  prompt: "Fix this checkout bug with a regression test; keep the change to the affected module."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: no-model-pinned-route
+    config:
+      contains:
+        - "fable-opus"
+        - "fable-sonnet"
+        - "fable-haiku"

package/evals/fable-mode/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-fable-mode-001
+name: Off-scope prompt must not trigger fable-mode
+description: A request outside fable-mode's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Answer a quick question about CSS specificity."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "fable"
+        - "orchestrator"

package/evals/fable-mode/tasks/positive-haiku-mode.yaml

@@ -0,0 +1,16 @@
+id: positive-fable-haiku-002
+name: Explicit Haiku mode routes mechanical work
+description: "Stage large, multi-source or multi-session tasks through a written plan, named fable agents, failable per-stage checks, and skeptical delivery review; use fable-opus, fable-sonnet, or fable-haiku for explicit model selection."
+tags:
+  - positive-trigger
+  - fable-haiku
+inputs:
+  prompt: "Fable on haiku: convert the repository's JSON fixtures to the new stable format and verify every output."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: routes-haiku
+    config:
+      contains:
+        - "haiku"

package/evals/fable-mode/tasks/positive-opus-mode.yaml

@@ -0,0 +1,16 @@
+id: positive-fable-opus-004
+name: Explicit Opus mode routes macro orchestration
+description: "Stage large, multi-source or multi-session tasks through a written plan, named fable agents, failable per-stage checks, and skeptical delivery review; use fable-opus, fable-sonnet, or fable-haiku for explicit model selection."
+tags:
+  - positive-trigger
+  - fable-opus
+inputs:
+  prompt: "Fable on opus: orchestrate the cross-service migration, delegate artifacts to named workers, and cold-verify the final architecture decision."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: routes-opus
+    config:
+      contains:
+        - "opus"

package/evals/fable-mode/tasks/positive-sonnet-mode.yaml

@@ -0,0 +1,16 @@
+id: positive-fable-sonnet-003
+name: Explicit Sonnet mode routes reasoning work
+description: "Stage large, multi-source or multi-session tasks through a written plan, named fable agents, failable per-stage checks, and skeptical delivery review; use fable-opus, fable-sonnet, or fable-haiku for explicit model selection."
+tags:
+  - positive-trigger
+  - fable-sonnet
+inputs:
+  prompt: "Fable on sonnect: synthesize the API research, implement the bounded adapter, and report the named verification command."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: routes-sonnet
+    config:
+      contains:
+        - "sonnet"

package/evals/fable-mode/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-fable-mode-001
+name: On-scope prompt triggers fable-mode
+description: "Stage large, multi-source or multi-session tasks through a written plan, named fable agents, failable per-stage checks, and skeptical delivery review; use fable-opus, fable-sonnet, or fable-haiku for explicit model selection."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Execute this large multi-phase refactor of the persistence layer using parallel sub-agent workers."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "worker"

package/evals/find-skills/eval.yaml

@@ -0,0 +1,26 @@
+name: find-skills-routing-eval
+description: Trigger precision eval for the find-skills skill.
+skill: find-skills
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did find-skills trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/find-skills/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-find-skills-001
+name: Off-scope prompt must not trigger find-skills
+description: A request outside find-skills's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Run the test suite and report failures."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "skills.sh"
+        - "npx skills"

package/evals/find-skills/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-find-skills-001
+name: On-scope prompt triggers find-skills
+description: "Search the agent-skills ecosystem (skills.sh / npx skills) when no installed skill fits; verify quality, apply ephemerally via temp cache by default, and install permanently via npx skills add only with explicit user approval"
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Find a skill that can generate mermaid sequence diagrams from code."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "skill"

package/evals/git-commit/eval.yaml

@@ -0,0 +1,26 @@
+name: git-commit-routing-eval
+description: Trigger precision eval for the git-commit skill.
+skill: git-commit
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did git-commit trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/git-commit/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-git-commit-001
+name: Off-scope prompt must not trigger git-commit
+description: A request outside git-commit's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Explain the difference between git rebase and git merge."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "Conventional Commit"
+        - "staged"

package/evals/git-commit/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-git-commit-001
+name: On-scope prompt triggers git-commit
+description: "Generate Angular-style commit messages after verifying the environment, submodules, and staged files; use for explicit commit requests or concluded task phases."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Commit these staged changes with a conventional commit message following our house style."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "commit"