harness-everything 0.2.2 → 0.4.0

package/evals/to-spec/eval.yaml

@@ -0,0 +1,26 @@
+name: to-spec-routing-eval
+description: Trigger precision eval for the to-spec skill.
+skill: to-spec
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did to-spec trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/to-spec/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-to-spec-001
+name: Off-scope prompt must not trigger to-spec
+description: A request outside to-spec's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Chat about weekend plans for a bit."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "feature-spec"
+        - "spec"

package/evals/to-spec/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-to-spec-001
+name: On-scope prompt triggers to-spec
+description: "Turn the conversation into a written spec — feature PRD, CLI/API reference, schema doc, or dev doc, whichever fits — and publish it per this repos projectDocs framework; use when alignment is complete."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Turn this rough feature request into a full written spec with schema and CLI reference sections."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "spec"

package/evals/to-tickets/eval.yaml

@@ -0,0 +1,26 @@
+name: to-tickets-routing-eval
+description: Trigger precision eval for the to-tickets skill.
+skill: to-tickets
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did to-tickets trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/to-tickets/tasks/negative-trigger.yaml

@@ -0,0 +1,15 @@
+id: negative-to-tickets-001
+name: Off-scope prompt must not trigger to-tickets
+description: A request outside to-tickets's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Optimize this slow SQL query."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "ticket"

package/evals/to-tickets/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-to-tickets-001
+name: On-scope prompt triggers to-tickets
+description: "Break a to-spec feature-spec, a plan, or a conversation into tracer-bullet tickets with declared blocking edges — one file per ticket locally or issues on a real tracker via projectDocs. Use when an approved spec needs dependency-aware ordering."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Break this approved spec into implementation tickets and publish them to our tracker format."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "ticket"

package/evals/todo-driven-workflow/eval.yaml

@@ -0,0 +1,26 @@
+name: todo-driven-workflow-eval
+description: Trigger precision eval for the native TODO and Markdown checklist workflow.
+skill: todo-driven-workflow
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did multi-step work get tracked through native TODO or Markdown checklist state transitions?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/todo-driven-workflow/tasks/negative-single-edit.yaml

@@ -0,0 +1,16 @@
+id: negative-single-edit-001
+name: Trivial single edit skips todo machinery
+description: Trivial single-file edits and quick questions without implementation must not be wrapped in a todo state machine.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Rename the variable 'usrName' to 'userName' in src/utils/format.js."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "stage map"
+        - "creating a todo list"

package/evals/todo-driven-workflow/tasks/positive-multi-step.yaml

@@ -0,0 +1,15 @@
+id: positive-multi-step-001
+name: Multi-step task gets tracked as todos
+description: "Enforce a deliberate step-by-step execution loop for complex multi-step or multi-file tasks (Tier 2/3), one sub-task in-progress at a time. USE FOR: \"track a complex multi-step task as todos\", \"work through a multi-file refactor step by step\". DO NOT USE FOR: \"trivial single-file edits\", \"quick questions without implementation\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Migrate all REST endpoints in src/api to the new error-handling middleware, update the tests, and refresh the docs. Keep track of progress as you go."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "todo"

package/evals/todo-driven-workflow/tasks/positive-refactor-flow.yaml

@@ -0,0 +1,15 @@
+id: positive-refactor-flow-002
+name: Multi-file refactor tracked step by step
+description: "Enforce a deliberate step-by-step execution loop for complex multi-step or multi-file tasks (Tier 2/3), one sub-task in-progress at a time. USE FOR: \"track a complex multi-step task as todos\", \"work through a multi-file refactor step by step\". DO NOT USE FOR: \"trivial single-file edits\", \"quick questions without implementation\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Work through this multi-file refactor step by step: move src/utils/date.js helpers into src/lib/time and update every importer."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "todo"

package/evals/using-git-worktrees/eval.yaml

@@ -0,0 +1,26 @@
+name: using-git-worktrees-routing-eval
+description: Trigger precision eval for the using-git-worktrees skill.
+skill: using-git-worktrees
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did using-git-worktrees trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/using-git-worktrees/tasks/negative-trigger.yaml

@@ -0,0 +1,15 @@
+id: negative-using-git-worktrees-001
+name: Off-scope prompt must not trigger using-git-worktrees
+description: A request outside using-git-worktrees's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "What time is it right now?"
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "worktree"

package/evals/using-git-worktrees/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-using-git-worktrees-001
+name: On-scope prompt triggers using-git-worktrees
+description: "Use when starting feature work needing isolation or before implementation plans - ensures an isolated workspace via native tools or git worktree fallback"
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Set up a git worktree so I can prototype in parallel without disturbing my checked-out branch."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "worktree"

package/evals/verification-loop/eval.yaml

@@ -0,0 +1,26 @@
+name: verification-loop-eval
+description: Trigger precision eval for the verification-loop skill.
+skill: verification-loop
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did the verification loop run objective checks before completion claims?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/verification-loop/tasks/negative-discussion.yaml

@@ -0,0 +1,16 @@
+id: negative-discussion-001
+name: Design discussion must not launch verification loop
+description: A design discussion with no completed work to verify, or planning without code changes, needs no gates.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Sketch two possible approaches for caching API responses and list trade-offs. Don't write any code yet."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "running the verification report"
+        - "verification passed"

package/evals/verification-loop/tasks/positive-pre-ship.yaml

@@ -0,0 +1,15 @@
+id: positive-pre-ship-verify-001
+name: Pre-completion request runs verification loop
+description: "Run objective verification gates (build, types, lint, tests, security scan) before claiming work done. USE FOR: \"run the full verification loop\", \"verify this is done before delivery\". DO NOT USE FOR: \"design discussions with no completed work\", \"planning without code changes\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "You said the refactor is finished. Run the full verification loop and show me the evidence before we call this done."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "verification"

package/evals/verify-before-claim/eval.yaml

@@ -0,0 +1,26 @@
+name: verify-before-claim-routing-eval
+description: Trigger precision eval for the verify-before-claim skill.
+skill: verify-before-claim
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did verify-before-claim trigger exactly when it should?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/verify-before-claim/tasks/negative-trigger.yaml

@@ -0,0 +1,16 @@
+id: negative-verify-before-claim-001
+name: Off-scope prompt must not trigger verify-before-claim
+description: A request outside verify-before-claim's DO NOT USE FOR scope must leave it dormant.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Tell me about your overall architecture."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "verification"
+        - "evidence"

package/evals/verify-before-claim/tasks/positive-trigger.yaml

@@ -0,0 +1,15 @@
+id: positive-verify-before-claim-001
+name: On-scope prompt triggers verify-before-claim
+description: "Verify external framework/API claims and unmeasured perf/cost estimates against an authoritative source before asserting them. Use when stating how an external framework, SDK, CLI, or API behaves, or quoting numbers not produced by a real run; output is a cited claim, a real measurement, or a labeled estimate."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "Before you say this is done, verify every changed behavior actually passes — no claiming from reading code."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "verify"

package/evals/zoom-out/eval.yaml

@@ -0,0 +1,26 @@
+name: zoom-out-eval
+description: Trigger precision eval for the zoom-out circuit breaker skill.
+skill: zoom-out
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+metrics:
+  - name: task_completion
+    weight: 0.7
+    threshold: 0.8
+    description: Did zoom-out trigger on repeated-failure/stuck signals and stay off trivial tasks?
+  - name: efficiency
+    weight: 0.3
+    threshold: 0.7
+    description: Did the skill stay within behavior limits?
+graders:
+  - type: behavior
+    name: token-budget
+    config:
+      max_tokens: 1200
+tasks:
+  - "tasks/*.yaml"

package/evals/zoom-out/tasks/negative-trivial-edit.yaml

@@ -0,0 +1,16 @@
+id: negative-trivial-edit-001
+name: Trivial edit must not trigger reflection
+description: Routine single-failure debugging and greenfield planning without failures must not trip the circuit breaker.
+tags:
+  - negative-trigger
+inputs:
+  prompt: "Fix the typo in README.md line 3: change 'recieve' to 'receive'."
+expected:
+  should_trigger: false
+graders:
+  - type: text
+    name: not-contains-keywords
+    config:
+      contains:
+        - "circuit breaker tripped"
+        - "reflection report"

package/evals/zoom-out/tasks/positive-repeated-failure.yaml

@@ -0,0 +1,15 @@
+id: positive-repeated-failure-001
+name: Repeated failure triggers reflection
+description: "Reflect-first circuit breaker - after 3 failures, stop edits, rebuild the full picture with read-only tools, write a fact-check report, then resume or escalate. USE FOR: \"you are stuck in a loop\", \"the same error keeps failing\", \"rethink your assumptions\". DO NOT USE FOR: \"routine single-failure debugging\", \"greenfield planning without failures\"."
+tags:
+  - positive-trigger
+inputs:
+  prompt: "You are stuck in a loop: the same error keeps failing with this TypeError even after three fixes. Step back and rethink your assumptions before trying again."
+expected:
+  should_trigger: true
+graders:
+  - type: text
+    name: contains-keywords
+    config:
+      contains:
+        - "zoom-out"

package/fable-discipline/SKILL.md

@@ -1,8 +1,10 @@
 ---
 name: fable-discipline
-description: Shadow guard for fable-mode to prevent context bloat and maintain physical boundaries.
-author: Miya Daniel | Harness Core Team
-version: 0.2.0
+description: Enforce fable-mode's context discipline as a shadow guard. Use when running staged multi-agent loops to prevent context bloat, cap per-stage output size, keep each stage within its physical token boundaries, and stop agents from reading files outside the current stage's scope.
+license: Apache-2.0
+metadata:
+  author: Miya Daniel
+  version: 0.4.0
 ---
 
 # Fable Discipline (Macro Task Discipline & Safety Net)
@@ -11,28 +13,29 @@ version: 0.2.0
 
 | Component | Specification |
 | :--- | :--- |
-| **Trigger / Input** | Runs continuously in the background for the entire duration `fable-mode` is active — not a standalone, one-shot trigger. |
-| **Expected Output** | Periodic state compaction after each milestone; one Git commit per independently functioning logic block; a state manifest handed off between sub-agents at every handoff point. |
-| **State Mutations** | None of its own — it constrains how `fable-mode` and `create-agent-launcher` mutate state (commit cadence, handoff manifests, read-size limits). |
-| **Enforcement Gate** | **HALTS execution immediately** (not a retry) if build errors show a divergent trend (more fixes → more breakage), and forces `zoom-out` instead of continued patching. |
-
-This skill acts as the shadow guard for `fable-mode`. As long as `fable-mode` is active, this skill MUST run in the background to prevent large tasks from spiraling out of control.
-
-## 1. Anti-Context Bloat Defense
-When executing large architectural tasks, conversation logs expand rapidly, leading to model attention loss (hallucinations or forgetting the original intent).
-- **Strategic Compact**: After completing each core milestone, forcefully summarize the current state and decisions, discarding unnecessary past conversation details.
-- **Avoid Broad Reads**: Prohibited from using broad Regex searches without precise conditions or reading irrelevant files over 1000 lines.
-
-## 2. Strict Physical Boundaries
-- **Environment Isolation**: Before modifying any core architecture, ensure you are clearly aware of the Current Working Directory (CWD).
-- **Atomic Commits**: You are NOT allowed to modify dozens of files at once before testing. For every independently functioning logic block completed, advise the human to create a Git Commit as a safe save point.
-
-## 3. Agent Handoff Protocol
-When `fable-mode` spawns and switches between different sub-agents via `create-agent-launcher`, strict handoff discipline must be observed:
-- **State Manifest**: The previous agent MUST leave a clear state record (e.g., what APIs were completed, expected inputs/outputs).
-- **Contract Testing**: The very first step for the succeeding agent is to verify if the state manifest left by the previous agent is correct.
-
-## 4. Ultimate Circuit Breaker: Indefinite Halt
-- Large tasks easily generate the sunk cost fallacy of "I think it's almost fixed, let me try one more time."
-- If the system's Build Errors exhibit a divergent trend (the more you fix, the more it breaks), **HALT EXECUTION IMMEDIATELY**.
-- Call `zoom-out`, organize the error dependency graph for the human, and seek architectural-level refactoring advice.
+| **Trigger / Input** | Background shadow for the whole duration `fable-mode` is active — not a standalone one-shot trigger. |
+| **Expected Output** | State compaction after milestones; one Git commit per independent logic block; a state manifest at every sub-agent handoff. |
+| **State Mutations** | None of its own — constrains how `fable-mode` and `multi-agent-workspace` mutate state. |
+| **Enforcement Gate** | **HALTS execution immediately** if build errors diverge (more fixes → more breakage); forces `zoom-out` instead of continued patching. |
+
+Shadow guard for `fable-mode`: MUST run in the background for its whole duration.
+
+## USE FOR:
+- Long architectural tasks under `fable-mode`
+- Compacting state after milestones
+- Sub-agent handoffs via `multi-agent-workspace`
+- Halting on diverging build errors
+
+## DO NOT USE FOR:
+- Small tasks where `fable-mode` is not active
+- Standalone one-shot invocation (background shadow skill)
+- Routine single-file edits
+
+## Core Rules
+
+1. **Anti-context bloat**: After each milestone, compact state and decisions; drop unneeded history. No broad regex without precise conditions; no reading irrelevant files over 1000 lines.
+2. **Physical boundaries**: Know your CWD before touching core architecture; one Git commit per independent logic block.
+3. **Agent handoff**: Outgoing agents MUST leave a state manifest (completed APIs, expected inputs/outputs); incoming agents MUST verify it first.
+4. **Circuit breaker**: If build errors diverge, HALT, call `zoom-out`, and map the error dependency graph for the human.
+
+Deep dive: <this-skill-dir>/references/discipline-rules.md

package/fable-discipline/references/discipline-rules.md

@@ -0,0 +1,22 @@
+# Fable Discipline — Full Detail
+
+This skill acts as the shadow guard for `fable-mode`. As long as `fable-mode` is active, this skill MUST run in the background to prevent large tasks from spiraling out of control.
+
+## 1. Anti-Context Bloat Defense
+When executing large architectural tasks, conversation logs expand rapidly, leading to model attention loss (hallucinations or forgetting the original intent).
+- **Strategic Compact**: After completing each core milestone, forcefully summarize the current state and decisions, discarding unnecessary past conversation details.
+- **Avoid Broad Reads**: Prohibited from using broad Regex searches without precise conditions or reading irrelevant files over 1000 lines.
+
+## 2. Strict Physical Boundaries
+- **Environment Isolation**: Before modifying any core architecture, ensure you are clearly aware of the Current Working Directory (CWD).
+- **Atomic Commits**: You are NOT allowed to modify dozens of files at once before testing. For every independently functioning logic block completed, advise the human to create a Git Commit as a safe save point.
+
+## 3. Agent Handoff Protocol
+When `fable-mode` spawns and switches between different sub-agents via `multi-agent-workspace`, strict handoff discipline must be observed:
+- **State Manifest**: The previous agent MUST leave a clear state record (e.g., what APIs were completed, expected inputs/outputs).
+- **Contract Testing**: The very first step for the succeeding agent is to verify if the state manifest left by the previous agent is correct.
+
+## 4. Ultimate Circuit Breaker: Indefinite Halt
+- Large tasks easily generate the sunk cost fallacy of "I think it's almost fixed, let me try one more time."
+- If the system's Build Errors exhibit a divergent trend (the more you fix, the more it breaks), **HALT EXECUTION IMMEDIATELY**.
+- Call `zoom-out`, organize the error dependency graph for the human, and seek architectural-level refactoring advice.