harness-everything 0.2.1 → 0.3.6

package/evals/grill-me/eval.yaml

@@ -0,0 +1,26 @@
+name: grill-me-routing-eval
+description: Trigger precision eval for the grill-me skill.
+skill: grill-me
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did grill-me trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/grill-me/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-grill-me-001
+name: Off-scope prompt must not trigger grill-me
+description: A request outside grill-me's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Summarize the contents of docs/philosophy.md."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "grill"
+        - "interview"

package/evals/grill-me/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-grill-me-001
+name: On-scope prompt triggers grill-me
+description: "Acts as a relentless challenger to stress-test plans and architectures by interrogating one question at a time, finding loopholes, combating AI sycophancy, updating the CONTEXT.md glossary in real time, and handing off resolved decisions to to-spec for formal specs/ADRs."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Grill me on my startup idea until the requirements are airtight before we build anything."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "question"

package/evals/grill-with-docs/eval.yaml

@@ -0,0 +1,26 @@
+name: grill-with-docs-routing-eval
+description: Trigger precision eval for the grill-with-docs skill.
+skill: grill-with-docs
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did grill-with-docs trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/grill-with-docs/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-grill-with-docs-001
+name: Off-scope prompt must not trigger grill-with-docs
+description: A request outside grill-with-docs's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Bump all dependency versions to latest."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "CONTEXT.md"
+        - "ADR"

package/evals/grill-with-docs/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-grill-with-docs-001
+name: On-scope prompt triggers grill-with-docs
+description: "Challenge the plan against the existing domain model, sharpen terminology, and update CONTEXT.md and ADRs inline as decisions crystallise; use when stress-testing a plan against documented language and decisions."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Interview me about this domain and persist what we establish into CONTEXT.md and ADRs."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "ADR"

package/evals/harness-everything/eval.yaml

@@ -0,0 +1,26 @@
+name: harness-everything-router-eval
+description: Tier routing precision eval for the harness-everything entry router.
+skill: harness-everything
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did the router emit the correct tier (1/2/3) for each prompt class?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/harness-everything/tasks/negative-non-software.yaml

@@ -0,0 +1,16 @@
+id: negative-non-software-001
+name: Non-software question bypasses routing
+description: Chat or general Q&A, and non-software writing, bypass the router entirely — no Checkpoint.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "What's the difference between latte and cappuccino?"
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "Routing Checkpoint"
+        - "Tier"

package/evals/harness-everything/tasks/negative-skill-indicated.yaml

@@ -0,0 +1,15 @@
+id: negative-skill-indicated-001
+name: Explicit skill indication bypasses routing
+description: A skill is already indicated, so the router must not re-route or print a Checkpoint.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Use the git-commit skill to commit my staged changes."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "Routing Checkpoint"

package/evals/harness-everything/tasks/positive-tier1.yaml

@@ -0,0 +1,16 @@
+id: positive-trivial-tier1-001
+name: Trivial edit routes to Tier 1
+description: "Triage new software requests into Tier 1/2/3 before work begins. USE FOR: \"triage a new software task\", \"route this coding request\". DO NOT USE FOR: \"chat or general Q&A\", \"non-software writing\", \"a skill is already indicated\"."
+tags:
+  - positive-trigger
+  - tier-1
+inputs:
+  prompt: "Triage this new software task: fix the typo 'lorem ipsum' to 'lorem ipsum dolor' in docs/intro.md."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "Tier 1"

package/evals/harness-everything/tasks/positive-tier2.yaml

@@ -0,0 +1,16 @@
+id: positive-standard-tier2-001
+name: Standard bug fix routes to Tier 2
+description: "Triage new software requests into Tier 1/2/3 before work begins. USE FOR: \"triage a new software task\", \"route this coding request\". DO NOT USE FOR: \"chat or general Q&A\", \"non-software writing\", \"a skill is already indicated\"."
+tags:
+  - positive-trigger
+  - tier-2
+inputs:
+  prompt: "Fix this bug: the checkout total double-counts coupon discounts when two coupons apply."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "Tier 2"

package/evals/harness-everything/tasks/positive-tier3.yaml

@@ -0,0 +1,16 @@
+id: positive-macro-tier3-001
+name: Architectural task routes to Tier 3
+description: "Triage new software requests into Tier 1/2/3 before work begins. USE FOR: \"triage a new software task\", \"route this coding request\". DO NOT USE FOR: \"chat or general Q&A\", \"non-software writing\", \"a skill is already indicated\"."
+tags:
+  - positive-trigger
+  - tier-3
+inputs:
+  prompt: "Route this coding request: split our monolith into two services (orders and billing) with an event bus between them; this touches deployment, data migration, and every team's code."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "Tier 3"

package/evals/improve-codebase-architecture/eval.yaml

@@ -0,0 +1,26 @@
+name: improve-codebase-architecture-routing-eval
+description: Trigger precision eval for the improve-codebase-architecture skill.
+skill: improve-codebase-architecture
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did improve-codebase-architecture trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/improve-codebase-architecture/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-improve-codebase-architecture-001
+name: Off-scope prompt must not trigger improve-codebase-architecture
+description: A request outside improve-codebase-architecture's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Update the copyright year across files."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "refactor plan"
+        - "architecture report"

package/evals/improve-codebase-architecture/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-improve-codebase-architecture-001
+name: On-scope prompt triggers improve-codebase-architecture
+description: "Discovers and transforms shallow or bloated modules into deep, SRP-compliant ones by splitting oversized objects/functions, extracting seams, and decoupling interfaces."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Improve the architecture of this tangled billing module and produce a report of proposed refactorings."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "architecture"

package/evals/install-cognitive-os/eval.yaml

@@ -0,0 +1,26 @@
+name: install-cognitive-os-routing-eval
+description: Trigger precision eval for the install-cognitive-os skill.
+skill: install-cognitive-os
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did install-cognitive-os trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/install-cognitive-os/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-install-cognitive-os-001
+name: Off-scope prompt must not trigger install-cognitive-os
+description: A request outside install-cognitive-os's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Delete the node_modules directory."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "Cognitive Law"
+        - "cognitive loop"

package/evals/install-cognitive-os/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-install-cognitive-os-001
+name: On-scope prompt triggers install-cognitive-os
+description: "Defines the fundamental physical laws of behavior — Discover, Think, Try, Summarize, Record — for all Agents. Use this loop before acting on any task to ground progress in verified tool output."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Install the cognitive OS loop into this workspace so evidence assertion and state handoff are enforced."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "cognitive"

package/evals/multi-agent-workspace/eval.yaml

@@ -0,0 +1,26 @@
+name: multi-agent-workspace-routing-eval
+description: Trigger precision eval for the multi-agent-workspace skill.
+skill: multi-agent-workspace
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did multi-agent-workspace trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/multi-agent-workspace/tasks/negative-trigger.yaml

@@ -0,0 +1,15 @@
+id: negative-multi-agent-workspace-001
+name: Off-scope prompt must not trigger multi-agent-workspace
+description: A request for an ordinary single-agent code fix must leave the multi-agent workspace skill dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Fix the typo in src/utils.js and run the unit test."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "multi-agent-workspace"

package/evals/multi-agent-workspace/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-multi-agent-workspace-001
+name: On-scope prompt triggers multi-agent-workspace
+description: "Scaffold a verified multi-agent workspace and select bounded specialists from an external agency-agents catalog without vendoring the full roster."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Scaffold a six-zone multi-agent workspace from our agency-agents source and select engineering specialists for a bounded launcher."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "multi-agent-workspace"

package/evals/repo-docs/eval.yaml

@@ -0,0 +1,26 @@
+name: repo-docs-routing-eval
+description: Trigger precision eval for the repo-docs skill.
+skill: repo-docs
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did repo-docs trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/repo-docs/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-repo-docs-001
+name: Off-scope prompt must not trigger repo-docs
+description: A request outside repo-docs's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Fix the failing unit test in payment.test.js."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "template"
+        - "AGENTS.md"

package/evals/repo-docs/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-repo-docs-001
+name: On-scope prompt triggers repo-docs
+description: "Generate or update precise, reader-oriented README.md and AGENTS.md based on actual project scans and smart merging of existing files. Use when a project lacks documentation or needs refreshed agent onboarding docs."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Generate AGENTS.md and README documentation for this repo from its actual structure."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "AGENTS.md"

package/evals/rewrite-commits/eval.yaml

@@ -0,0 +1,26 @@
+name: rewrite-commits-routing-eval
+description: Trigger precision eval for the rewrite-commits skill.
+skill: rewrite-commits
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did rewrite-commits trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/rewrite-commits/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-rewrite-commits-001
+name: Off-scope prompt must not trigger rewrite-commits
+description: A request outside rewrite-commits's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Show me the diff of the working tree."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "rebase"
+        - "rewrite"

package/evals/rewrite-commits/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-rewrite-commits-001
+name: On-scope prompt triggers rewrite-commits
+description: "Rewrite past Git history to comply with Angular Style conventions. Use when cleaning unpushed commits before a PR, squashing fixups, rewording messages, or reordering history safely with abort on conflict."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Rewrite my last five commits into clean conventional commits before opening the PR."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "rewrite"

package/evals/security-review/eval.yaml

@@ -0,0 +1,26 @@
+name: security-review-routing-eval
+description: Trigger precision eval for the security-review skill.
+skill: security-review
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did security-review trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/security-review/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-security-review-001
+name: Off-scope prompt must not trigger security-review
+description: A request outside security-review's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Add a dark mode toggle to the settings page."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "STRIDE"
+        - "threat model"

package/evals/security-review/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-security-review-001
+name: On-scope prompt triggers security-review
+description: "Conducts STRIDE threat modeling, scans secrets via audit scripts, and hardens code against OWASP Top 10 risks with three-tier boundary controls. Use when implementing auth, handling input/uploads, managing secrets, or auditing code before deployment."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Do a security review of the new payments endpoint, checking OWASP patterns and secrets exposure."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "OWASP"

package/evals/self-evolve/eval.yaml

@@ -0,0 +1,26 @@
+name: self-evolve-routing-eval
+description: Trigger precision eval for the self-evolve skill.
+skill: self-evolve
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did self-evolve trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/self-evolve/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-self-evolve-001
+name: Off-scope prompt must not trigger self-evolve
+description: A request outside self-evolve's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Print hello world to the console."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "RULES.md"
+        - "dynamic skill"

package/evals/self-evolve/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-self-evolve-001
+name: On-scope prompt triggers self-evolve
+description: "Extract root causes from resolved problems and persist them as defensive memory rules or dynamic skills; use after struggles, zoom-outs, or explicit request."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Reflect on the root cause of that recurring test failure and persist the lesson as a workspace rule or dynamic skill."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "lesson"