harness-everything 0.2.2 → 0.4.0

package/evals/improve-codebase-architecture/eval.yaml

@@ -0,0 +1,26 @@
+name: improve-codebase-architecture-routing-eval
+description: Trigger precision eval for the improve-codebase-architecture skill.
+skill: improve-codebase-architecture
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did improve-codebase-architecture trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/improve-codebase-architecture/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-improve-codebase-architecture-001
+name: Off-scope prompt must not trigger improve-codebase-architecture
+description: A request outside improve-codebase-architecture's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Update the copyright year across files."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "refactor plan"
+        - "architecture report"

package/evals/improve-codebase-architecture/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-improve-codebase-architecture-001
+name: On-scope prompt triggers improve-codebase-architecture
+description: "Discovers and transforms shallow or bloated modules into deep, SRP-compliant ones by splitting oversized objects/functions, extracting seams, and decoupling interfaces."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Improve the architecture of this tangled billing module and produce a report of proposed refactorings."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "architecture"

package/evals/install-cognitive-os/eval.yaml

@@ -0,0 +1,26 @@
+name: install-cognitive-os-routing-eval
+description: Trigger precision eval for the install-cognitive-os skill.
+skill: install-cognitive-os
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did install-cognitive-os trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/install-cognitive-os/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-install-cognitive-os-001
+name: Off-scope prompt must not trigger install-cognitive-os
+description: A request outside install-cognitive-os's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Delete the node_modules directory."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "Cognitive Law"
+        - "cognitive loop"

package/evals/install-cognitive-os/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-install-cognitive-os-001
+name: On-scope prompt triggers install-cognitive-os
+description: "Explain or explicitly apply the Harness cognitive policy: Discover, Think, Try, Summarize, Record. It provides cross-cutting behavior without requiring this skill to be selected before domain skills."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Install the cognitive OS loop into this workspace so evidence assertion and state handoff are enforced."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "cognitive"

package/evals/multi-agent-workspace/eval.yaml

@@ -0,0 +1,26 @@
+name: multi-agent-workspace-routing-eval
+description: Trigger precision eval for the multi-agent-workspace skill.
+skill: multi-agent-workspace
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did multi-agent-workspace trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/multi-agent-workspace/tasks/negative-trigger.yaml

@@ -0,0 +1,15 @@
+id: negative-multi-agent-workspace-001
+name: Off-scope prompt must not trigger multi-agent-workspace
+description: A request for an ordinary single-agent code fix must leave the multi-agent workspace skill dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Fix the typo in src/utils.js and run the unit test."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "multi-agent-workspace"

package/evals/multi-agent-workspace/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-multi-agent-workspace-001
+name: On-scope prompt triggers multi-agent-workspace
+description: "Scaffold a verified multi-agent workspace and select bounded specialists from an external agency-agents catalog without vendoring the full roster."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Scaffold a six-zone multi-agent workspace from our agency-agents source and select engineering specialists for a bounded launcher."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "multi-agent-workspace"

package/evals/repo-docs/eval.yaml

@@ -0,0 +1,26 @@
+name: repo-docs-routing-eval
+description: Trigger precision eval for the repo-docs skill.
+skill: repo-docs
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did repo-docs trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/repo-docs/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-repo-docs-001
+name: Off-scope prompt must not trigger repo-docs
+description: A request outside repo-docs's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Fix the failing unit test in payment.test.js."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "template"
+        - "AGENTS.md"

package/evals/repo-docs/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-repo-docs-001
+name: On-scope prompt triggers repo-docs
+description: "Generate or update precise, reader-oriented README.md and AGENTS.md based on actual project scans and smart merging of existing files. Use when a project lacks documentation or needs refreshed agent onboarding docs."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Generate AGENTS.md and README documentation for this repo from its actual structure."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "AGENTS.md"

package/evals/rewrite-commits/eval.yaml

@@ -0,0 +1,26 @@
+name: rewrite-commits-routing-eval
+description: Trigger precision eval for the rewrite-commits skill.
+skill: rewrite-commits
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did rewrite-commits trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/rewrite-commits/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-rewrite-commits-001
+name: Off-scope prompt must not trigger rewrite-commits
+description: A request outside rewrite-commits's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Show me the diff of the working tree."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "rebase"
+        - "rewrite"

package/evals/rewrite-commits/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-rewrite-commits-001
+name: On-scope prompt triggers rewrite-commits
+description: "Rewrite past Git history to comply with Angular Style conventions. Use when cleaning unpushed commits before a PR, squashing fixups, rewording messages, or reordering history safely with abort on conflict."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Rewrite my last five commits into clean conventional commits before opening the PR."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "rewrite"

package/evals/security-review/eval.yaml

@@ -0,0 +1,26 @@
+name: security-review-routing-eval
+description: Trigger precision eval for the security-review skill.
+skill: security-review
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did security-review trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/security-review/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-security-review-001
+name: Off-scope prompt must not trigger security-review
+description: A request outside security-review's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Add a dark mode toggle to the settings page."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "STRIDE"
+        - "threat model"

package/evals/security-review/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-security-review-001
+name: On-scope prompt triggers security-review
+description: "Conducts STRIDE threat modeling, scans secrets via audit scripts, and hardens code against OWASP Top 10 risks with three-tier boundary controls. Use when implementing auth, handling input/uploads, managing secrets, or auditing code before deployment."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Do a security review of the new payments endpoint, checking OWASP patterns and secrets exposure."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "OWASP"

package/evals/self-evolve/eval.yaml

@@ -0,0 +1,26 @@
+name: self-evolve-routing-eval
+description: Trigger precision eval for the self-evolve skill.
+skill: self-evolve
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did self-evolve trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/self-evolve/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-self-evolve-001
+name: Off-scope prompt must not trigger self-evolve
+description: A request outside self-evolve's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Print hello world to the console."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "RULES.md"
+        - "dynamic skill"

package/evals/self-evolve/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-self-evolve-001
+name: On-scope prompt triggers self-evolve
+description: "Extract root causes from resolved problems and persist them as defensive memory rules or dynamic skills; use after struggles, zoom-outs, or explicit request."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Reflect on the root cause of that recurring test failure and persist the lesson as a workspace rule or dynamic skill."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "lesson"

package/evals/skill-creator/eval.yaml

@@ -0,0 +1,26 @@
+name: skill-creator-routing-eval
+description: Trigger precision eval for the skill-creator skill.
+skill: skill-creator
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did skill-creator trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/skill-creator/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-skill-creator-001
+name: Off-scope prompt must not trigger skill-creator
+description: A request outside skill-creator's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Run the linters over the project."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "SKILL.md"
+        - "frontmatter"

package/evals/skill-creator/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-skill-creator-001
+name: On-scope prompt triggers skill-creator
+description: "Author, audit, and refactor SKILL.md files against one quality bar — the Skill Contract format. Use when creating a new skill, refactoring a SKILL.md, checking overlap, or packaging an insight as a dynamic skill."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Create a new skill for database migration workflows with proper references and testing workflow."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "skill"

package/evals/skill-style/eval.yaml

@@ -0,0 +1,26 @@
+name: skill-style-routing-eval
+description: Trigger precision eval for the skill-style skill.
+skill: skill-style
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did skill-style trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/skill-style/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-skill-style-001
+name: Off-scope prompt must not trigger skill-style
+description: A request outside skill-style's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Deploy the app to staging."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "style guide"
+        - "progressive disclosure"

package/evals/skill-style/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-skill-style-001
+name: On-scope prompt triggers skill-style
+description: "Apply Harness style guidelines when writing or refactoring SKILL.md files. Use when creating a new skill, reviewing skill structure, enforcing naming and frontmatter conventions, or standardizing tone, formatting, and progressive disclosure across the skill catalog."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Review this SKILL.md against the house style guide and fix deviations."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "style"

package/evals/tdd/eval.yaml

@@ -0,0 +1,26 @@
+name: tdd-eval
+description: Trigger precision eval for the TDD (RED/GREEN/REFACTOR) skill.
+skill: tdd
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did Tier 2 feature work start from a failing test and respect the RED/GREEN/REFACTOR gates?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/tdd/tasks/negative-trivia.yaml

@@ -0,0 +1,16 @@
+id: negative-trivia-001
+name: Non-code question must not enter TDD loop
+description: Docs-only work with no testable logic must not start a RED/GREEN cycle; macro planning or scaffolding belongs to fable-mode.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "In one sentence, what does the RED phase of test-driven development mean?"
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "writing a failing test now"
+        - "entering RED"

package/evals/tdd/tasks/positive-bugfix.yaml

@@ -0,0 +1,15 @@
+id: positive-bugfix-tdd-002
+name: Bug fix requests a regression test
+description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GREEN/REFACTOR) backed by terminal test evidence. USE FOR: \"implement a feature test-first\", \"fix a bug with a regression test\", \"refactor safely under tests\". DO NOT USE FOR: \"macro planning or scaffolding\", \"docs-only work with no testable logic\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Fix a bug with a regression test: the discount calculation rounds wrong for prices ending in .99."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "regression"

package/evals/tdd/tasks/positive-feature.yaml

@@ -0,0 +1,16 @@
+id: positive-feature-tdd-001
+name: Standard feature requests TDD enforcement
+description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GREEN/REFACTOR) backed by terminal test evidence. USE FOR: \"implement a feature test-first\", \"fix a bug with a regression test\", \"refactor safely under tests\". DO NOT USE FOR: \"macro planning or scaffolding\", \"docs-only work with no testable logic\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Implement a feature test-first: add input validation to the user signup function — reject emails without '@' and passwords shorter than 8 characters."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "RED"
+        - "failing test"

package/evals/tdd/tasks/positive-refactor.yaml

@@ -0,0 +1,15 @@
+id: positive-refactor-tdd-003
+name: Refactoring under green tests
+description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GREEN/REFACTOR) backed by terminal test evidence. USE FOR: \"implement a feature test-first\", \"fix a bug with a regression test\", \"refactor safely under tests\". DO NOT USE FOR: \"macro planning or scaffolding\", \"docs-only work with no testable logic\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Refactor safely under tests: extract the pricing logic in src/checkout into its own module without changing behavior."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "REFACTOR"