harness-everything 0.2.1 → 0.3.6

package/evals/skill-creator/eval.yaml

@@ -0,0 +1,26 @@
+name: skill-creator-routing-eval
+description: Trigger precision eval for the skill-creator skill.
+skill: skill-creator
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did skill-creator trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/skill-creator/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-skill-creator-001
+name: Off-scope prompt must not trigger skill-creator
+description: A request outside skill-creator's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Run the linters over the project."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "SKILL.md"
+        - "frontmatter"

package/evals/skill-creator/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-skill-creator-001
+name: On-scope prompt triggers skill-creator
+description: "Author, audit, and refactor SKILL.md files against one quality bar — the Skill Contract format. Use when creating a new skill, refactoring a SKILL.md, checking overlap, or packaging an insight as a dynamic skill."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Create a new skill for database migration workflows with proper references and testing workflow."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "skill"

package/evals/skill-style/eval.yaml

@@ -0,0 +1,26 @@
+name: skill-style-routing-eval
+description: Trigger precision eval for the skill-style skill.
+skill: skill-style
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did skill-style trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/skill-style/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-skill-style-001
+name: Off-scope prompt must not trigger skill-style
+description: A request outside skill-style's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Deploy the app to staging."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "style guide"
+        - "progressive disclosure"

package/evals/skill-style/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-skill-style-001
+name: On-scope prompt triggers skill-style
+description: "Apply Harness style guidelines when writing or refactoring SKILL.md files. Use when creating a new skill, reviewing skill structure, enforcing naming and frontmatter conventions, or standardizing tone, formatting, and progressive disclosure across the skill catalog."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Review this SKILL.md against the house style guide and fix deviations."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "style"

package/evals/tdd/eval.yaml

@@ -0,0 +1,26 @@
+name: tdd-eval
+description: Trigger precision eval for the TDD (RED/GREEN/REFACTOR) skill.
+skill: tdd
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did Tier 2 feature work start from a failing test and respect the RED/GREEN/REFACTOR gates?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/tdd/tasks/negative-trivia.yaml

@@ -0,0 +1,16 @@
+id: negative-trivia-001
+name: Non-code question must not enter TDD loop
+description: Docs-only work with no testable logic must not start a RED/GREEN cycle; macro planning or scaffolding belongs to fable-mode.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "In one sentence, what does the RED phase of test-driven development mean?"
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "writing a failing test now"
+        - "entering RED"

package/evals/tdd/tasks/positive-bugfix.yaml

@@ -0,0 +1,15 @@
+id: positive-bugfix-tdd-002
+name: Bug fix requests a regression test
+description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GREEN/REFACTOR) backed by terminal test evidence. USE FOR: \"implement a feature test-first\", \"fix a bug with a regression test\", \"refactor safely under tests\". DO NOT USE FOR: \"macro planning or scaffolding\", \"docs-only work with no testable logic\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Fix a bug with a regression test: the discount calculation rounds wrong for prices ending in .99."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "regression"

package/evals/tdd/tasks/positive-feature.yaml

@@ -0,0 +1,16 @@
+id: positive-feature-tdd-001
+name: Standard feature requests TDD enforcement
+description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GREEN/REFACTOR) backed by terminal test evidence. USE FOR: \"implement a feature test-first\", \"fix a bug with a regression test\", \"refactor safely under tests\". DO NOT USE FOR: \"macro planning or scaffolding\", \"docs-only work with no testable logic\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Implement a feature test-first: add input validation to the user signup function — reject emails without '@' and passwords shorter than 8 characters."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "RED"
+        - "failing test"

package/evals/tdd/tasks/positive-refactor.yaml

@@ -0,0 +1,15 @@
+id: positive-refactor-tdd-003
+name: Refactoring under green tests
+description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GREEN/REFACTOR) backed by terminal test evidence. USE FOR: \"implement a feature test-first\", \"fix a bug with a regression test\", \"refactor safely under tests\". DO NOT USE FOR: \"macro planning or scaffolding\", \"docs-only work with no testable logic\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Refactor safely under tests: extract the pricing logic in src/checkout into its own module without changing behavior."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "REFACTOR"

package/evals/to-spec/eval.yaml

@@ -0,0 +1,26 @@
+name: to-spec-routing-eval
+description: Trigger precision eval for the to-spec skill.
+skill: to-spec
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did to-spec trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/to-spec/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-to-spec-001
+name: Off-scope prompt must not trigger to-spec
+description: A request outside to-spec's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Chat about weekend plans for a bit."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "feature-spec"
+        - "spec"

package/evals/to-spec/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-to-spec-001
+name: On-scope prompt triggers to-spec
+description: "Turn the conversation into a written spec — feature PRD, CLI/API reference, schema doc, or dev doc, whichever fits — and publish it per this repos projectDocs framework; use when alignment is complete."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Turn this rough feature request into a full written spec with schema and CLI reference sections."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "spec"

package/evals/to-tickets/eval.yaml

@@ -0,0 +1,26 @@
+name: to-tickets-routing-eval
+description: Trigger precision eval for the to-tickets skill.
+skill: to-tickets
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did to-tickets trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/to-tickets/tasks/negative-trigger.yaml

@@ -0,0 +1,15 @@
+id: negative-to-tickets-001
+name: Off-scope prompt must not trigger to-tickets
+description: A request outside to-tickets's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Optimize this slow SQL query."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "ticket"

package/evals/to-tickets/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-to-tickets-001
+name: On-scope prompt triggers to-tickets
+description: "Break a to-spec feature-spec, a plan, or a conversation into tracer-bullet tickets with declared blocking edges — one file per ticket locally or issues on a real tracker via projectDocs. Use when an approved spec needs dependency-aware ordering."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Break this approved spec into implementation tickets and publish them to our tracker format."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "ticket"

package/evals/todo-driven-workflow/eval.yaml

@@ -0,0 +1,26 @@
+name: todo-driven-workflow-eval
+description: Trigger precision eval for the native TODO and Markdown checklist workflow.
+skill: todo-driven-workflow
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did multi-step work get tracked through native TODO or Markdown checklist state transitions?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/todo-driven-workflow/tasks/negative-single-edit.yaml

@@ -0,0 +1,16 @@
+id: negative-single-edit-001
+name: Trivial single edit skips todo machinery
+description: Trivial single-file edits and quick questions without implementation must not be wrapped in a todo state machine.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Rename the variable 'usrName' to 'userName' in src/utils/format.js."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "stage map"
+        - "creating a todo list"

package/evals/todo-driven-workflow/tasks/positive-multi-step.yaml

@@ -0,0 +1,15 @@
+id: positive-multi-step-001
+name: Multi-step task gets tracked as todos
+description: "Enforce a deliberate step-by-step execution loop for complex multi-step or multi-file tasks (Tier 2/3), one sub-task in-progress at a time. USE FOR: \"track a complex multi-step task as todos\", \"work through a multi-file refactor step by step\". DO NOT USE FOR: \"trivial single-file edits\", \"quick questions without implementation\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Migrate all REST endpoints in src/api to the new error-handling middleware, update the tests, and refresh the docs. Keep track of progress as you go."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "todo"

package/evals/todo-driven-workflow/tasks/positive-refactor-flow.yaml

@@ -0,0 +1,15 @@
+id: positive-refactor-flow-002
+name: Multi-file refactor tracked step by step
+description: "Enforce a deliberate step-by-step execution loop for complex multi-step or multi-file tasks (Tier 2/3), one sub-task in-progress at a time. USE FOR: \"track a complex multi-step task as todos\", \"work through a multi-file refactor step by step\". DO NOT USE FOR: \"trivial single-file edits\", \"quick questions without implementation\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Work through this multi-file refactor step by step: move src/utils/date.js helpers into src/lib/time and update every importer."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "todo"

package/evals/using-git-worktrees/eval.yaml

@@ -0,0 +1,26 @@
+name: using-git-worktrees-routing-eval
+description: Trigger precision eval for the using-git-worktrees skill.
+skill: using-git-worktrees
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did using-git-worktrees trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/using-git-worktrees/tasks/negative-trigger.yaml

@@ -0,0 +1,15 @@
+id: negative-using-git-worktrees-001
+name: Off-scope prompt must not trigger using-git-worktrees
+description: A request outside using-git-worktrees's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "What time is it right now?"
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "worktree"

package/evals/using-git-worktrees/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-using-git-worktrees-001
+name: On-scope prompt triggers using-git-worktrees
+description: "Use when starting feature work needing isolation or before implementation plans - ensures an isolated workspace via native tools or git worktree fallback"
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Set up a git worktree so I can prototype in parallel without disturbing my checked-out branch."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "worktree"

package/evals/verification-loop/eval.yaml

@@ -0,0 +1,26 @@
+name: verification-loop-eval
+description: Trigger precision eval for the verification-loop skill.
+skill: verification-loop
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did the verification loop run objective checks before completion claims?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/verification-loop/tasks/negative-discussion.yaml

@@ -0,0 +1,16 @@
+id: negative-discussion-001
+name: Design discussion must not launch verification loop
+description: A design discussion with no completed work to verify, or planning without code changes, needs no gates.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Sketch two possible approaches for caching API responses and list trade-offs. Don't write any code yet."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "running the verification report"
+        - "verification passed"

package/evals/verification-loop/tasks/positive-pre-ship.yaml

@@ -0,0 +1,15 @@
+id: positive-pre-ship-verify-001
+name: Pre-completion request runs verification loop
+description: "Run objective verification gates (build, types, lint, tests, security scan) before claiming work done. USE FOR: \"run the full verification loop\", \"verify this is done before delivery\". DO NOT USE FOR: \"design discussions with no completed work\", \"planning without code changes\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "You said the refactor is finished. Run the full verification loop and show me the evidence before we call this done."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "verification"

package/evals/verify-before-claim/eval.yaml

@@ -0,0 +1,26 @@
+name: verify-before-claim-routing-eval
+description: Trigger precision eval for the verify-before-claim skill.
+skill: verify-before-claim
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did verify-before-claim trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/verify-before-claim/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-verify-before-claim-001
+name: Off-scope prompt must not trigger verify-before-claim
+description: A request outside verify-before-claim's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Tell me about your overall architecture."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "verification"
+        - "evidence"

package/evals/verify-before-claim/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-verify-before-claim-001
+name: On-scope prompt triggers verify-before-claim
+description: "Verify external framework/API claims and unmeasured perf/cost estimates against an authoritative source before asserting them. Use when stating how an external framework, SDK, CLI, or API behaves, or quoting numbers not produced by a real run; output is a cited claim, a real measurement, or a labeled estimate."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Before you say this is done, verify every changed behavior actually passes — no claiming from reading code."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "verify"

package/evals/zoom-out/eval.yaml

@@ -0,0 +1,26 @@
+name: zoom-out-eval
+description: Trigger precision eval for the zoom-out circuit breaker skill.
+skill: zoom-out
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did zoom-out trigger on repeated-failure/stuck signals and stay off trivial tasks?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/zoom-out/tasks/negative-trivial-edit.yaml

@@ -0,0 +1,16 @@
+id: negative-trivial-edit-001
+name: Trivial edit must not trigger reflection
+description: Routine single-failure debugging and greenfield planning without failures must not trip the circuit breaker.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Fix the typo in README.md line 3: change 'recieve' to 'receive'."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "circuit breaker tripped"
+        - "reflection report"

package/evals/zoom-out/tasks/positive-repeated-failure.yaml

@@ -0,0 +1,15 @@
+id: positive-repeated-failure-001
+name: Repeated failure triggers reflection
+description: "Reflect-first circuit breaker - after 3 failures, stop edits, rebuild the full picture with read-only tools, write a fact-check report, then resume or escalate. USE FOR: \"you are stuck in a loop\", \"the same error keeps failing\", \"rethink your assumptions\". DO NOT USE FOR: \"routine single-failure debugging\", \"greenfield planning without failures\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "You are stuck in a loop: the same error keeps failing with this TypeError even after three fixes. Step back and rethink your assumptions before trying again."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "zoom-out"