harness-everything 0.2.2 → 0.4.0

package/environment-detection/scripts/preflight.js

@@ -1,93 +1,103 @@
-#!/usr/bin/env node
-const { execSync } = require('child_process');
-const os = require('os');
-const fs = require('fs');
-const path = require('path');
-
-function checkCmd(cmd) {
-  try {
-    if (os.platform() === 'win32') {
-      execSync(`where ${cmd}`, { stdio: 'ignore' });
-    } else {
-      execSync(`which ${cmd}`, { stdio: 'ignore' });
-    }
-    return true;
-  } catch (e) {
-    try {
-      execSync(`${cmd} --version`, { stdio: 'ignore' });
-      return true;
-    } catch (e2) {
-      return false;
-    }
-  }
-}
-
-console.log("# [Environment Preflight Report]");
-console.log(`- **Operating System**: ${os.platform()} (${os.release()} ${os.arch()})`);
-
-// Shell Detection
-let shell = process.env.SHELL || process.env.COMSPEC || "Unknown";
-let shellName = "Unknown Shell";
-
-if (shell.toLowerCase().includes("bash")) {
-  shellName = "Git Bash / Bash";
-} else if (shell.toLowerCase().includes("powershell") || shell.toLowerCase().includes("pwsh") || process.env.PSModulePath) {
-  shellName = "PowerShell";
-} else if (shell.toLowerCase().includes("cmd.exe")) {
-  shellName = "Windows Command Prompt (CMD)";
-} else {
-  shellName = shell;
-}
-
-console.log(`- **Active Shell**: ${shellName} (Path: \`${shell}\`)`);
-console.log(`- **Terminal Program**: ${process.env.TERM_PROGRAM || "Standard Terminal"}`);
-
-// CLI Tools Check
-const tools = ['git', 'docker', 'docker-compose', 'node', 'pnpm', 'npm', 'python', 'pip'];
-const availableTools = [];
-for (const tool of tools) {
-  if (checkCmd(tool)) {
-    availableTools.push(tool);
-  }
-}
-console.log(`- **Available CLI Tools**: ${availableTools.join(', ') || 'None'}`);
-
-// Sandbox Capabilities Detection
-const hasDocker = availableTools.includes('docker');
-const hasSandboxExec = checkCmd('sandbox-exec');
-const sandboxCapabilities = [];
-if (hasSandboxExec) sandboxCapabilities.push('macOS Seatbelt (sandbox-exec)');
-if (hasDocker) sandboxCapabilities.push('Docker Containers');
-if (os.platform() === 'win32') {
-  sandboxCapabilities.push('Windows AppContainer (Native)');
-}
-console.log(`- **Sandbox Capabilities**: ${sandboxCapabilities.join(', ') || 'No isolation sandbox detected'}`);
-
-// Repo Status
-try {
-  const branch = execSync('git branch --show-current', { encoding: 'utf8' }).trim();
-  console.log(`- **Git Workspace**: Detected (Branch: \`${branch}\`)`);
-} catch (e) {
-  console.log(`- **Git Workspace**: No git repository detected in ${process.cwd()}`);
-}
-
-// Recommended Syntax Guardrails
-console.log("\n## [Syntax & Command Guardrails for this Environment]");
-if (os.platform() === 'win32') {
-  if (shellName.includes("Bash")) {
-    console.log("- **Path Handling**: ALWAYS use `/` slashes for directory paths in commands (e.g., `scripts/run.js`). Avoid raw backslashes `\\` as they act as escape chars.");
-    console.log("- **Environment Variables**: Use `$VAR` syntax. Do NOT use `%VAR%` or `$env:VAR`.");
-    console.log("- **Command Chaining**: Use `&&` for sequence, `|` for piping.");
-    console.log("- **Avoid Native Windows Commands**: Do NOT use `dir`, `copy`, or `del`. Use bash standard equivalents `ls`, `cp`, `rm`.");
-  } else if (shellName.includes("PowerShell")) {
-    console.log("- **Path Handling**: Slashes `/` or `\\` are generally acceptable.");
-    console.log("- **Environment Variables**: Use `$env:VAR` syntax.");
-    console.log("- **Command Chaining**: Use `;` or `&&` (if PowerShell 7+). Avoid bash-specific commands like `grep`, `awk`, `sed` unless explicitly piping.");
-  } else {
-    console.log("- **Path Handling**: Use `\\` for directories.");
-    console.log("- **Environment Variables**: Use `%VAR%`.");
-    console.log("- **Chaining**: Use `&`.");
-  }
-} else {
-  console.log("- **Unix Native**: Standard Bash/Zsh syntax applies. Use relative paths, pipeline chaining, and standard utilities.");
-}
+#!/usr/bin/env node
+const { execSync } = require('child_process');
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+
+function checkCmd(cmd) {
+  try {
+    if (os.platform() === 'win32') {
+      execSync(`where "${cmd}"`, { stdio: 'ignore' });
+    } else {
+      execSync(`which "${cmd}"`, { stdio: 'ignore' });
+    }
+    return true;
+  } catch (e) {
+    try {
+      execSync(`${cmd} --version`, { stdio: 'ignore' });
+      return true;
+    } catch (e2) {
+      return false;
+    }
+  }
+}
+
+console.log("# [Environment Preflight Report]");
+console.log(`- **Operating System**: ${os.platform()} (${os.release()} ${os.arch()})`);
+
+// Shell Detection
+const isGitBash = !!(
+  process.env.MSYSTEM ||
+  process.env.BASH_VERSION ||
+  (process.env.SHELL && process.env.SHELL.toLowerCase().includes("bash"))
+);
+const isPowerShell = !!(
+  process.env.PSModulePath ||
+  (process.env.SHELL && (process.env.SHELL.toLowerCase().includes("powershell") || process.env.SHELL.toLowerCase().includes("pwsh")))
+);
+
+let shell = process.env.SHELL || (isGitBash ? "bash" : (isPowerShell ? "powershell" : process.env.COMSPEC)) || "Unknown";
+let shellName = "Unknown Shell";
+
+if (isGitBash || shell.toLowerCase().includes("bash")) {
+  shellName = "Git Bash / Bash";
+} else if (isPowerShell) {
+  shellName = "PowerShell";
+} else if (shell.toLowerCase().includes("cmd.exe")) {
+  shellName = "Windows Command Prompt (CMD)";
+} else {
+  shellName = shell;
+}
+
+console.log(`- **Active Shell**: ${shellName} (Path: \`${shell}\`)`);
+console.log(`- **Terminal Program**: ${process.env.TERM_PROGRAM || "Standard Terminal"}`);
+
+// CLI Tools Check
+const tools = ['git', 'docker', 'docker-compose', 'node', 'pnpm', 'npm', 'python', 'pip'];
+const availableTools = [];
+for (const tool of tools) {
+  if (checkCmd(tool)) {
+    availableTools.push(tool);
+  }
+}
+console.log(`- **Available CLI Tools**: ${availableTools.join(', ') || 'None'}`);
+
+// Sandbox Capabilities Detection
+const hasDocker = availableTools.includes('docker');
+const hasSandboxExec = checkCmd('sandbox-exec');
+const sandboxCapabilities = [];
+if (hasSandboxExec) sandboxCapabilities.push('macOS Seatbelt (sandbox-exec)');
+if (hasDocker) sandboxCapabilities.push('Docker Containers');
+if (os.platform() === 'win32') {
+  sandboxCapabilities.push('Windows AppContainer (Native)');
+}
+console.log(`- **Sandbox Capabilities**: ${sandboxCapabilities.join(', ') || 'No isolation sandbox detected'}`);
+
+// Repo Status
+try {
+  const branch = execSync('git branch --show-current', { encoding: 'utf8' }).trim();
+  console.log(`- **Git Workspace**: Detected (Branch: \`${branch}\`)`);
+} catch (e) {
+  console.log(`- **Git Workspace**: No git repository detected in ${process.cwd()}`);
+}
+
+// Recommended Syntax Guardrails
+console.log("\n## [Syntax & Command Guardrails for this Environment]");
+if (os.platform() === 'win32') {
+  if (shellName.includes("Bash")) {
+    console.log("- **Path Handling**: ALWAYS use `/` slashes for directory paths in commands (e.g., `scripts/run.js`). Avoid raw backslashes `\\` as they act as escape chars.");
+    console.log("- **Environment Variables**: Use `$VAR` syntax. Do NOT use `%VAR%` or `$env:VAR`.");
+    console.log("- **Command Chaining**: Use `&&` for sequence, `|` for piping.");
+    console.log("- **Avoid Native Windows Commands**: Do NOT use `dir`, `copy`, or `del`. Use bash standard equivalents `ls`, `cp`, `rm`.");
+  } else if (shellName.includes("PowerShell")) {
+    console.log("- **Path Handling**: Slashes `/` or `\\` are generally acceptable.");
+    console.log("- **Environment Variables**: Use `$env:VAR` syntax.");
+    console.log("- **Command Chaining**: Use `;` or `&&` (if PowerShell 7+). Avoid bash-specific commands like `grep`, `awk`, `sed` unless explicitly piping.");
+  } else {
+    console.log("- **Path Handling**: Use `\\` for directories.");
+    console.log("- **Environment Variables**: Use `%VAR%`.");
+    console.log("- **Chaining**: Use `&`.");
+  }
+} else {
+  console.log("- **Unix Native**: Standard Bash/Zsh syntax applies. Use relative paths, pipeline chaining, and standard utilities.");
+}

package/eval-framework/fixtures/bad-frontmatter-colon/SKILL.md

@@ -0,0 +1,14 @@
+---
+name: bad-frontmatter-colon
+description: Bad unquoted colon: here
+version: 0.3.4
+metadata:
+  author: Miya Daniel
+  version: 0.3.4
+---
+
+## USE FOR:
+Testing negative control for unquoted YAML colon
+
+## DO NOT USE FOR:
+Production use

package/eval-framework/fixtures/duplicate-desc-skill-a/SKILL.md

@@ -0,0 +1,14 @@
+---
+name: duplicate-desc-skill-a
+description: Test skill with identical description for collision detection
+version: 0.3.4
+metadata:
+  author: Miya Daniel
+  version: 0.3.4
+---
+
+## USE FOR:
+Testing negative control for duplicate description detection
+
+## DO NOT USE FOR:
+Production use

package/eval-framework/fixtures/duplicate-desc-skill-b/SKILL.md

@@ -0,0 +1,14 @@
+---
+name: duplicate-desc-skill-b
+description: Test skill with identical description for collision detection
+version: 0.3.4
+metadata:
+  author: Miya Daniel
+  version: 0.3.4
+---
+
+## USE FOR:
+Testing negative control for duplicate description detection
+
+## DO NOT USE FOR:
+Production use

package/eval-framework/fixtures/missing-delimiter/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: missing-delimiter
+description: Missing end delimiter
+version: 0.3.4
+metadata:
+  author: Miya Daniel
+  version: 0.3.4
+
+## USE FOR:
+Testing negative control for missing delimiter
+
+## DO NOT USE FOR:
+Production use

package/eval-framework/fixtures/over-budget-skill/SKILL.md

@@ -0,0 +1,36 @@
+---
+name: over-budget-skill
+description: Over budget skill for token limit testing
+version: 0.3.4
+metadata:
+  author: Miya Daniel
+  version: 0.3.4
+---
+
+## USE FOR:
+This skill is intentionally over the word budget to test the token limit enforcement gate. It contains a very long description and body that exceeds the 500 word hard limit. The purpose of this skill is to verify that the consistency-check.js script correctly detects when a SKILL.md file exceeds the token budget and fails the check. This is a negative control fixture that must fail the token budget gate.
+
+The skill description is intentionally verbose to push the word count over the limit. In a real skill, this would be a routing surface issue where the description is too long and bloats token budgets for agent routing. The token budget gate in consistency-check.js uses a word-count proxy (1.55 tokens per word) to estimate token usage and fails when the hard limit of 500 tokens is exceeded.
+
+This negative control ensures that the gate actually works and catches over-budget skills. Without this fixture, a gate that silently passes everything would be indistinguishable from a working gate.
+
+Additional content to push word count over 500 words. This skill contains extensive documentation about how the token budget enforcement works in the Harness system. The gate uses a simple word-count heuristic where each word is estimated to be approximately 1.55 tokens. This means 330 words roughly equals 511 tokens, which exceeds the 500 token hard limit. The gate has two thresholds: a warning at 400 words (80% of hard limit) and a hard failure at 500 words.
+
+The word counting is performed on the body of the SKILL.md file after removing the frontmatter. This means the frontmatter fields like name, description, version, and metadata are not counted toward the word budget. Only the markdown content in the body sections (USE FOR, DO NOT USE FOR, and any additional reference content) contributes to the word count.
+
+To create a skill that exceeds the budget, this fixture includes many paragraphs of explanatory text. Each paragraph adds to the word count. The test verifies that when a skill exceeds 500 words, the consistency-check.js script fails with an appropriate error message indicating the number of words over the limit.
+
+This is a comprehensive test of the token budget enforcement mechanism. The gate is designed to be a local early-warning system that catches token budget issues before they reach CI, where waza tokens check would also catch them. By having both local and CI gates, we ensure that token budget violations are caught at multiple stages.
+
+Production systems should never have skills that exceed the token budget because long descriptions bloat the routing context and make agent selection less efficient. The routing system uses skill descriptions as the primary signal for skill selection, so keeping them concise and focused is essential for good routing performance.
+
+Additional paragraphs to ensure we exceed the five hundred word threshold. Each sentence here adds multiple words to the total count. We need to reach at least five hundred and one words to trigger the hard failure gate. The consistency check will then report the exact number of words over the limit and fail the build. This is the expected behavior for the negative control test.
+
+More content to push the word count higher. The word counting algorithm splits on whitespace and filters out empty strings. This means each space-separated token counts as a word. Punctuation attached to words does not affect the count. We are now adding several more sentences to ensure we cross the five hundred word boundary with comfortable margin.
+
+Even more text to guarantee the word count exceeds the hard limit. The test fixture must reliably fail the token budget check every time it runs. Flaky tests that sometimes pass and sometimes fail are worse than no tests at all. By making the content significantly over the limit, we ensure consistent test results.
+
+Final paragraph to push us well over the five hundred word target. At this point we should have more than enough words to trigger the hard failure. The consistency check will output an error showing the exact word count and how many words exceed the limit. This confirms the gate is working correctly.
+
+## DO NOT USE FOR:
+Production use - this is a test fixture only

package/eval-harness/SKILL.md

@@ -1,59 +1,42 @@
 ---
 name: eval-harness
-description: Objectively evaluates AI agent performance based on correctness, resource efficiency, and anti-loop focus.
-author: Miya Daniel | Harness Core Team
-version: 0.2.0
+description: Evaluate AI agent performance across correctness, token efficiency, anti-loop focus, and environment awareness; save a 4-dimension scorecard. Use when asked to run a benchmark or score an execution log or conversation history.
+license: Apache-2.0
+metadata:
+  author: Miya Daniel
+  version: 0.4.0
 ---
 
-# Eval Harness (Automated Performance & Reasoning Evaluation)
+# Eval Harness
 
-## 📋 Skill Contract
+Evaluate an agent log with a four-dimension scorecard.
+
+## Skill Contract
 
 | Component | Specification |
 | :--- | :--- |
-| **Trigger / Input** | User requests "run benchmark", "score conversation log", or "compare Vanilla vs Harness differences" — an execution log or the current conversation history. |
-| **Expected Output** | A 4-dimension scorecard (Correctness & Factuality, Token & Step Efficiency, Anti-loop & Focus, Environment & Tool Awareness), each scored 0-10 with rationale. |
-| **State Mutations** | Writes a generated Markdown scorecard to the project's `evals/` directory via `scripts/evaluate.js`. |
-| **Enforcement Gate** | MUST execute `scripts/evaluate.js` with the four scores and insight text — a verbally-stated score without running the script does not count as output. |
-
-This skill is used to objectively evaluate the performance of an AI Agent when "executing test tasks".
-Triggered when the user requests "run benchmark", "score conversation log", or "compare Vanilla vs Harness differences".
-
-## 1. Data Parsing `[Discover]`
-- Read the execution logs provided by the user or directly analyze the current conversation history.
-- Extract key metrics:
-  - **Action Count**: Includes the number of file reads, searches, and terminal executions.
-  - **Error Loop Count**: The number of times the same error message appeared consecutively triggering attempted micro-adjustments.
-  - **Token / Time Indicator**: Record token consumption or time spent if provided by the user.
-
-## 2. Scoring Rubric `[Think]`
-Please score based on the following four dimensions (out of 10 points each):
-
-### A. Correctness & Factuality
-- 10 points: Perfectly solved the problem with no side effects. Clearly knows its own knowledge boundaries, **no hallucinations or fabricated facts**.
-- 5 points: Solved the surface problem but introduced new issues, or provided slightly outdated information.
-- 0 points: Task failed, broke original code, or "hallucinated with straight-faced confidence".
-
-### B. Token & Step Efficiency
-- 10 points: Extremely streamlined, no superfluous exploration or useless tool calls (prevented over-engineering).
-- 5 points: Reasonable steps, but with minor unnecessary file reads.
-- 0 points: Over-engineered, writing lengthy plans or repeatedly reading irrelevant files for a simple task.
-
-### C. Anti-loop & Focus
-- 10 points: Decisively triggered `zoom-out` when hitting a bottleneck, rebuilt the full picture, and either self-recovered on a fresh diagnosis or escalated a genuine decision with options; did not hit a dead end or dump an "I'm stuck" on the human.
-- 5 points: Stopped after 3~5 attempts.
-- 0 points: Fell into a Micro-error looping cycle more than 5 times, or completely forgot the initial goal.
-
-### D. Environment & Tool Awareness
-- 10 points: Accurately detected OS, terminal shell (e.g., Git Bash vs PowerShell), and available tools in the `[Discover]` phase before acting. No syntax errors caused by wrong shell assumptions.
-- 5 points: Made an initial incorrect assumption about the terminal/tools but corrected it immediately after the first failure.
-- 0 points: Blindly threw commands (e.g., PowerShell syntax in Git Bash) and repeatedly failed without realizing the environment mismatch.
-
-## 3. Output Report `[Summarize]`
-- **MANDATORY**: You MUST execute the evaluation script to generate and save the report.
-- Find the `evaluate.js` script in the `scripts/` directory of this skill and run it via terminal:
-  ```bash
-  node path/to/eval-harness/scripts/evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> "<Your insights here>"
-  ```
-  *(Example: `node scripts/evaluate.js 10 5 10 10 "The agent successfully used zoom-out..."`)*
-- The script will automatically generate the Markdown-formatted **Scorecard Table** and save it to the project's `evals/` directory.
+| **Trigger / Input** | A benchmark request and an execution log or conversation history. |
+| **Expected Output** | A 0–10 scorecard for correctness, efficiency, anti-loop focus, and environment awareness. |
+| **State Mutations** | Writes the scorecard under `<workspace>/evals/` or `<workspace>/.github/harness-everything/evals/`. |
+| **Enforcement Gate** | Run `<this-skill-dir>/scripts/evaluate.js`; use the documented fallback only if it fails or is missing. |
+
+## Workflow
+
+1. Parse actions, errors, tokens, and timing from the log.
+2. Score all four dimensions using `<this-skill-dir>/references/scoring-rubric.md`.
+3. Run `node <this-skill-dir>/scripts/evaluate.js <A> <B> <C> <D> "insights"`.
+4. If the script fails, write `<workspace>/evals/scorecard.md` (or the `<workspace>/.github` fallback).
+
+## USE FOR:
+
+- Running a benchmark or scoring an agent execution.
+- Comparing Vanilla and Harness execution.
+- Generating a four-dimension scorecard.
+
+## DO NOT USE FOR:
+
+- Writing or fixing the evaluated code.
+- General review without scoring dimensions.
+- Tasks without an execution log.
+
+Deep dive: `<this-skill-dir>/references/scoring-rubric.md`

package/eval-harness/references/scoring-rubric.md

@@ -0,0 +1,68 @@
+# Eval Harness — Full Process & Scoring Rubric
+
+## Process & Evaluation Output Flow
+
+Follow the decision matrix below to evaluate performance and record scorecards:
+
+```mermaid
+flowchart TD
+    Start[Trigger: Benchmark / Evaluate Log] --> Parse[1. Parse Log: Actions, Loops, Tokens]
+    Parse --> Score[2. Calculate 4-Dimension Rubric Scores]
+
+    Score --> CheckScript{3. Is evaluate.js Executable?}
+
+    CheckScript -- Yes --> RunScript[Run node "<this-skill-dir>/scripts/evaluate.js" <scores> <insights>]
+    RunScript -- Exit 0 (Success) --> Done[Scorecard Saved to evals/]
+    RunScript -- Fails --> FileFallback
+
+    CheckScript -- No --> FileFallback{4. Markdown File Fallback}
+
+    FileFallback -- evals/ Directory Exists --> WriteEvals[Write to evals/scorecard.md]
+    FileFallback -- No evals/ Directory --> WritePlatform[Write to .github/harness-everything/evals/scorecard.md]
+
+    WriteEvals --> Done
+    WritePlatform --> Done
+```
+
+## 1. Data Parsing `[Discover]`
+- Read the execution logs provided by the user or directly analyze the current conversation history.
+- Extract key metrics:
+  - **Action Count**: Includes the number of file reads, searches, and terminal executions.
+  - **Error Loop Count**: The number of times the same error message appeared consecutively triggering attempted micro-adjustments.
+  - **Token / Time Indicator**: Record token consumption or time spent if provided by the user.
+
+## 2. Scoring Rubric `[Think]`
+
+Score based on the following four dimensions (out of 10 points each):
+
+### A. Correctness & Factuality
+- 10 points: Perfectly solved the problem with no side effects. Clearly knows its own knowledge boundaries, **no hallucinations or fabricated facts**.
+- 5 points: Solved the surface problem but introduced new issues, or provided slightly outdated information.
+- 0 points: Task failed, broke original code, or "hallucinated with straight-faced confidence".
+
+### B. Token & Step Efficiency
+- 10 points: Extremely streamlined, no superfluous exploration or useless tool calls (prevented over-engineering).
+- 5 points: Reasonable steps, but with minor unnecessary file reads.
+- 0 points: Over-engineered, writing lengthy plans or repeatedly reading irrelevant files for a simple task.
+
+### C. Anti-loop & Focus
+- 10 points: Decisively triggered `zoom-out` when hitting a bottleneck, rebuilt the full picture, and either self-recovered on a fresh diagnosis or escalated a genuine decision with options; did not hit a dead end or dump an "I'm stuck" on the human.
+- 5 points: Stopped after 3~5 attempts.
+- 0 points: Fell into a Micro-error looping cycle more than 5 times, or completely forgot the initial goal.
+
+### D. Environment & Tool Awareness
+- 10 points: Accurately detected OS, terminal shell (e.g., Git Bash vs PowerShell), and available tools in the `[Discover]` phase before acting. No syntax errors caused by wrong shell assumptions.
+- 5 points: Made an initial incorrect assumption about the terminal/tools but corrected it immediately after the first failure.
+- 0 points: Blindly threw commands (e.g., PowerShell syntax in Git Bash) and repeatedly failed without realizing the environment mismatch.
+
+## 3. Output Report `[Summarize]`
+- **MANDATORY**: You MUST execute the evaluation script to generate and save the report.
+- Find the `evaluate.js` script in the `scripts/` directory of this skill and run it via terminal:
+
+```bash
+node "<this-skill-dir>/scripts/evaluate.js" <scoreA> <scoreB> <scoreC> <scoreD> "<Your insights here>"
+```
+
+*(Example: `node scripts/evaluate.js 10 5 10 10 "The agent successfully used zoom-out..."`)* — run with `--help` for the scoring-dimension reference.
+
+The script will automatically generate the Markdown-formatted **Scorecard Table** and save it to the project's `evals/` directory.

package/eval-harness/scripts/evaluate.js

@@ -1,61 +1,79 @@
-#!/usr/bin/env node
-const fs = require('fs');
-const path = require('path');
-
-const args = process.argv.slice(2);
-if (args.length < 5) {
-  console.error("Usage: node evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> \"<insights>\"");
-  console.error("Scores should be 0, 5, or 10.");
-  process.exit(1);
-}
-
-const [scoreA, scoreB, scoreC, scoreD, ...insightParts] = args;
-const insights = insightParts.join(" ");
-
-const scores = {
-  "Correctness & Factuality": parseInt(scoreA, 10),
-  "Token & Step Efficiency": parseInt(scoreB, 10),
-  "Anti-loop & Focus": parseInt(scoreC, 10),
-  "Environment & Tool Awareness": parseInt(scoreD, 10)
-};
-
-let total = 0;
-let markdown = `## Harness Evaluation Report - ${new Date().toISOString()}\n\n| Dimension | Score |\n|---|---|\n`;
-
-for (const [dim, score] of Object.entries(scores)) {
-  if (isNaN(score)) {
-    console.error(`Error: Score for ${dim} is not a valid number.`);
-    process.exit(1);
-  }
-  markdown += `| ${dim} | ${score}/10 |\n`;
-  total += score;
-}
-
-markdown += `| **Total Score** | **${total}/40** |\n\n`;
-markdown += `### Insights\n${insights}\n\n---\n`;
-
-function getWorkspaceRoot() {
-  let dir = path.resolve(process.cwd());
-  while (dir !== path.parse(dir).root) {
-    if (fs.existsSync(path.join(dir, '.git'))) return dir;
-    dir = path.dirname(dir);
-  }
-  return process.cwd();
-}
-
-// Store the evaluations in an evals folder in the project root
-const evalDir = path.join(getWorkspaceRoot(), 'evals');
-if (!fs.existsSync(evalDir)) {
-  fs.mkdirSync(evalDir, { recursive: true });
-}
-
-const reportPath = path.join(evalDir, 'evaluation-reports.md');
-try {
-  fs.appendFileSync(reportPath, markdown, 'utf8');
-  console.log(`[Success] Evaluation report generated and saved to ${reportPath}`);
-  console.log('\n--- Report Preview ---\n');
-  console.log(markdown);
-} catch (err) {
-  console.error(`[Error] Failed to write evaluation report: ${err.message}`);
-  process.exit(1);
-}
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+let getWorkspaceRoot;
+try {
+  ({ getWorkspaceRoot } = require('../../scripts/lib/workspace'));
+} catch (err) {
+  getWorkspaceRoot = () => null;
+}
+
+const args = process.argv.slice(2);
+
+if (args[0] === '--help' || args[0] === '-h') {
+  console.log(`Score a Harness-guarded session and write an evaluation report.
+
+Usage: node evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> "<insights>"
+
+Four positional scores (0, 5, or 10 each), in this order:
+  scoreA  Correctness & Factuality
+  scoreB  Token & Step Efficiency
+  scoreC  Anti-loop & Focus
+  scoreD  Environment & Tool Awareness
+
+Followed by one free-text insights string (wrap in quotes).`);
+  process.exit(0);
+}
+
+if (args.length < 5) {
+  console.error("Usage: node evaluate.js <scoreA> <scoreB> <scoreC> <scoreD> \"<insights>\"");
+  console.error("Scores should be 0, 5, or 10.");
+  process.exit(1);
+}
+
+const [scoreA, scoreB, scoreC, scoreD, ...insightParts] = args;
+const insights = insightParts.join(" ");
+
+const scores = {
+  "Correctness & Factuality": parseInt(scoreA, 10),
+  "Token & Step Efficiency": parseInt(scoreB, 10),
+  "Anti-loop & Focus": parseInt(scoreC, 10),
+  "Environment & Tool Awareness": parseInt(scoreD, 10)
+};
+
+let total = 0;
+let markdown = `## Harness Evaluation Report - ${new Date().toISOString()}\n\n| Dimension | Score |\n|---|---|\n`;
+
+for (const [dim, score] of Object.entries(scores)) {
+  if (isNaN(score)) {
+    console.error(`Error: Score for ${dim} is not a valid number.`);
+    process.exit(1);
+  }
+  markdown += `| ${dim} | ${score}/10 |\n`;
+  total += score;
+}
+
+markdown += `| **Total Score** | **${total}/40** |\n\n`;
+markdown += `### Insights\n${insights}\n\n---\n`;
+
+// Store the evaluations in an evals folder in the project root
+const workspaceRoot = getWorkspaceRoot();
+if (!workspaceRoot) {
+  console.error('[eval-harness] Cannot write an evaluation report without a resolved git workspace.');
+  process.exit(1);
+}
+const evalDir = path.join(workspaceRoot, 'evals');
+if (!fs.existsSync(evalDir)) {
+  fs.mkdirSync(evalDir, { recursive: true });
+}
+
+const reportPath = path.join(evalDir, 'evaluation-reports.md');
+try {
+  fs.appendFileSync(reportPath, markdown, 'utf8');
+  console.log(`[Success] Evaluation report generated and saved to ${reportPath}`);
+  console.log('\n--- Report Preview ---\n');
+  console.log(markdown);
+} catch (err) {
+  console.error(`[Error] Failed to write evaluation report: ${err.message}`);
+  process.exit(1);
+}

package/evals/environment-detection/eval.yaml

@@ -0,0 +1,26 @@
+name: environment-detection-routing-eval
+description: Trigger precision eval for the environment-detection skill.
+skill: environment-detection
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did environment-detection trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"