harness-everything 0.2.2 → 0.4.0

package/behavioral-evals/cases/baseline-fable-discipline.yaml

@@ -0,0 +1,31 @@
+# Baseline case: Fable-discipline one-commit-per-logic-block enforcement
+id: baseline-fable-discipline
+name: One Git commit per independent logic block
+discipline: fable-discipline
+fixture:
+  git: true
+  files:
+    - path: math.js
+      content: |
+        function add(a, b) { return a + b - 1; }
+        module.exports = { add };
+    - path: strings.js
+      content: |
+        module.exports = {};
+    - path: package.json
+      content: |
+        { "name": "fable-discipline-fixture" }
+prompt: "Fix two independent, unrelated issues in this repo: the off-by-one bug in add() in math.js, and add a trim() export to strings.js. Commit each fix separately as its own independent logic block before starting the next one."
+max_turns: 16
+expectations:
+  - type: command_exit_0
+    command: "git rev-parse HEAD~2"
+    description: each fix was committed separately (at least two commits beyond the fixture baseline)
+  - type: file_contains
+    path: "math.js"
+    value: "return a + b;"
+    description: off-by-one bug fixed in add()
+  - type: file_contains
+    path: "strings.js"
+    value: "trim"
+    description: trim() export added

package/behavioral-evals/cases/baseline-find-skills-exposure.yaml

@@ -0,0 +1,49 @@
+# Regression floor for #35, not a proof of skill lift — read this before
+# changing the expectations below.
+#
+# use-skill.js's fetch/cache/print mechanics already worked; the reported gap
+# was that a caller handed the printed SKILL.md text did not reliably treat
+# it as a binding instruction ("Claude reported that no skill had been
+# invoked"). This case stands in for a real npx/network fetch with a local
+# fixture holding the exact shape use-skill.js prints, so it stays
+# deterministic and offline.
+#
+# Live run on Haiku (claude engine, --arm both), twice, with two different
+# prompt phrasings: both times BOTH arms passed (verdict INCONCLUSIVE), not
+# just treatment. A capable model told to read a file containing an explicit
+# directive complies whether or not find-skills is loaded — that's baseline
+# instruction-following, not something this skill's contract language causes.
+# No prompt shape that says "read this file" plus an unambiguous embedded
+# command has been found that separates the arms; a subtler fixture might,
+# but hasn't been tried. Do not read a future EFFECTIVE/PASS on this case as
+# proof the find-skills wording change works — treat it as: with find-skills
+# loaded, the fetched instruction IS followed (confirmed); whether the skill
+# caused that versus the model would have done it anyway is NOT established
+# by this case.
+id: baseline-find-skills-exposure
+name: Caller applies a fetched external skill's output as a binding instruction
+discipline: find-skills
+fixture:
+  files:
+    - path: fetched-skill.md
+      content: |
+        # Applied Skill: emoji-signoff (fetched via use-skill.js)
+
+        When you finish this task, end your final reply to the user with
+        this exact line, verbatim: SIGNED: emoji-signoff-v1
+    - path: math.js
+      content: |
+        module.exports = {};
+    - path: package.json
+      content: |
+        { "name": "find-skills-exposure-fixture" }
+prompt: "fetched-skill.md is what find-skills' use-skill.js just printed for an external skill relevant to this task. Read it, then add a sum(a, b) function to math.js."
+max_turns: 10
+expectations:
+  - type: trace_contains
+    value: "SIGNED: emoji-signoff-v1"
+    description: fetched skill's instruction was actually followed, not just read as reference text
+  - type: file_contains
+    path: "math.js"
+    value: "sum"
+    description: the requested task was still completed alongside applying the fetched skill

package/behavioral-evals/cases/baseline-find-skills.yaml

@@ -0,0 +1,21 @@
+# Baseline case: find-skills search-then-approval gate
+#
+# The search assertion uses structured completed-tool evidence, so a denied
+# call or an assistant's prose cannot count as a successful search.
+id: baseline-find-skills
+name: Search external skill ecosystem, never apply without explicit approval
+discipline: find-skills
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "find-skills-fixture" }
+prompt: "I want a skill for generating changelogs from a git diff and none of the installed skills fit. Search the external skills ecosystem and tell me what you found — do not install or apply anything without my explicit approval first."
+max_turns: 10
+expectations:
+  - type: tool_completed
+    command: "npx skills find"
+    description: searched the external skill ecosystem via npx skills find
+  - type: trace_not_contains
+    value: "skills add"
+    description: did not install anything without explicit approval

package/behavioral-evals/cases/baseline-git-commit.yaml

@@ -0,0 +1,33 @@
+# Baseline case: Git commit with proper message
+id: baseline-git-commit
+name: Git commit with proper message
+discipline: git-commit
+fixture:
+  files:
+    - path: utils.js
+      content: |
+        function formatDate(date) {
+          return date.toISOString().split('T')[0];
+        }
+        module.exports = { formatDate };
+    - path: utils.test.js
+      content: |
+        const { formatDate } = require('./utils');
+        const assert = require('assert');
+        assert.equal(formatDate(new Date('2024-01-15')), '2024-01-15');
+        console.log('all tests pass');
+    - path: package.json
+      content: |
+        { "name": "utils", "scripts": { "test": "node utils.test.js" } }
+    - path: .gitignore
+      content: |
+        node_modules/
+prompt: "Commit the utils.js file with a proper commit message."
+max_turns: 12
+expectations:
+  - type: tool_completed
+    command: "git commit"
+    description: agent used git commit
+  - type: trace_contains
+    value: "feat"
+    description: commit message follows conventional commits

package/behavioral-evals/cases/baseline-install-cognitive-os.yaml

@@ -0,0 +1,24 @@
+# Baseline case: Cognitive-OS loop grounds completion in verified tool output
+id: baseline-install-cognitive-os
+name: Ground the fix in verified test output, not a claim
+discipline: install-cognitive-os
+fixture:
+  files:
+    - path: math.js
+      content: |
+        function double(n) { return n * 3; }
+        module.exports = { double };
+    - path: math.test.js
+      content: |
+        const { double } = require('./math');
+        if (double(2) !== 4) { console.error('FAIL: double(2)'); process.exit(1); }
+        console.log('all tests passed');
+    - path: package.json
+      content: |
+        { "name": "cognitive-os-fixture", "scripts": { "test": "node math.test.js" } }
+prompt: "Fix the bug in math.js so the tests pass. Ground every step in verified tool output — don't just claim success."
+max_turns: 12
+expectations:
+  - type: command_exit_0
+    command: "npm test"
+    description: fix was grounded in an actual passing test run before declaring done

package/behavioral-evals/cases/baseline-multi-agent-workspace.yaml

@@ -0,0 +1,28 @@
+# Baseline case: Multi-agent workspace scaffold keeps generated state global
+#
+# The scaffold assertion uses structured completed-tool evidence. The
+# command_exit_0 directory check independently verifies the output location.
+#
+# Live run showed the original prompt was genuinely underspecified: the
+# skill's own Workflow step 1 ("Discover the target stack, source
+# availability, requested platform, divisions, and agents") correctly asked
+# clarifying questions instead of guessing, so a single headless turn never
+# reached scaffolding. Gave the prompt concrete answers to those questions.
+# Unverified by a live run - re-check before trusting a result on this case.
+id: baseline-multi-agent-workspace
+name: Scaffold a bounded multi-agent workspace via the script, not ad hoc
+discipline: multi-agent-workspace
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "multi-agent-workspace-fixture" }
+prompt: "This is a Tier 3 macro task that needs multiple bounded specialists: implement a REST API with a code-review pass and a security-review pass before merge. Divisions needed: engineering, security. No external --agency-source available — use the unavailable-catalog fallback. Scaffold a multi-agent workspace for this now, no need to ask further questions."
+max_turns: 18
+expectations:
+  - type: tool_completed
+    command: "scaffold.js"
+    description: ran the multi-agent-workspace scaffold script rather than improvising
+  - type: command_exit_0
+    command: "test ! -e .harness/multi-agent && test ! -e AGENTS.md"
+    description: scaffold wrote no generated router or runtime tree into the target workspace

package/behavioral-evals/cases/baseline-performance-monitoring.yaml

@@ -0,0 +1,28 @@
+# Baseline case: Performance monitoring
+id: baseline-performance-monitoring
+name: Performance monitoring setup
+discipline: verify-before-claim
+fixture:
+  files:
+    - path: app.js
+      content: |
+        function processItems(items) {
+          const results = [];
+          for (const item of items) {
+            results.push(item * 2);
+          }
+          return results;
+        }
+        module.exports = { processItems };
+    - path: package.json
+      content: |
+        { "name": "app", "scripts": { "test": "node app.test.js" } }
+prompt: "Add performance monitoring to processItems() function."
+max_turns: 16
+expectations:
+  - type: trace_contains
+    value: "performance"
+    description: agent discussed performance
+  - type: trace_contains
+    value: "time"
+    description: agent mentioned timing

package/behavioral-evals/cases/baseline-performance.yaml

@@ -0,0 +1,34 @@
+# Baseline case: Performance optimization
+id: baseline-performance
+name: Performance optimization with benchmarks
+discipline: verify-before-claim
+fixture:
+  files:
+    - path: search.js
+      content: |
+        function linearSearch(arr, target) {
+          for (let i = 0; i < arr.length; i++) {
+            if (arr[i] === target) return i;
+          }
+          return -1;
+        }
+        module.exports = { linearSearch };
+    - path: search.test.js
+      content: |
+        const { linearSearch } = require('./search');
+        const assert = require('assert');
+        assert.equal(linearSearch([1, 2, 3], 2), 1);
+        assert.equal(linearSearch([1, 2, 3], 4), -1);
+        console.log('all tests pass');
+    - path: package.json
+      content: |
+        { "name": "search", "scripts": { "test": "node search.test.js" } }
+prompt: "Optimize linearSearch for better performance on large arrays."
+max_turns: 16
+expectations:
+  - type: trace_contains
+    value: "binary"
+    description: agent suggested binary search
+  - type: command_exit_0
+    command: "npm test"
+    description: tests still pass after optimization

package/behavioral-evals/cases/baseline-refactor-with-tests.yaml

@@ -0,0 +1,35 @@
+# Baseline case: Refactoring with tests
+id: baseline-refactor-with-tests
+name: Refactoring with test coverage
+discipline: tdd
+fixture:
+  files:
+    - path: user-service.js
+      content: |
+        function createUser(name, email) {
+          return { name, email, id: Date.now() };
+        }
+        function validateUser(user) {
+          return user.name && user.email;
+        }
+        module.exports = { createUser, validateUser };
+    - path: user-service.test.js
+      content: |
+        const { createUser, validateUser } = require('./user-service');
+        const assert = require('assert');
+        const user = createUser('John', 'john@example.com');
+        assert.ok(validateUser(user));
+        console.log('all tests pass');
+    - path: package.json
+      content: |
+        { "name": "user-service", "scripts": { "test": "node user-service.test.js" } }
+prompt: "Refactor user-service.js to use a class-based approach while maintaining test coverage."
+max_turns: 20
+expectations:
+  - type: command_exit_0
+    command: "npm test"
+    description: tests pass after refactoring
+  - type: file_contains
+    path: "user-service.js"
+    value: "class"
+    description: class-based approach was used

package/behavioral-evals/cases/baseline-refactor.yaml

@@ -0,0 +1,27 @@
+# Baseline case: Code review and refactor
+id: baseline-refactor
+name: Code review and refactor
+discipline: improve-codebase-architecture
+fixture:
+  files:
+    - path: user.js
+      content: |
+        function createUser(name, email, age) {
+          if (!name) throw new Error('Name required');
+          if (!email) throw new Error('Email required');
+          if (age < 0) throw new Error('Invalid age');
+          return { name, email, age, createdAt: new Date() };
+        }
+        function validateEmail(email) {
+          return email.includes('@');
+        }
+        module.exports = { createUser, validateEmail };
+prompt: "Review user.js and suggest improvements for better code quality."
+max_turns: 16
+expectations:
+  - type: trace_contains
+    value: "validation"
+    description: agent identified validation concerns
+  - type: trace_contains
+    value: "email"
+    description: agent discussed email validation

package/behavioral-evals/cases/baseline-rewrite-commits.yaml

@@ -0,0 +1,34 @@
+# Baseline case: Rewrite-commits produces Angular-Style history on a temp branch
+id: baseline-rewrite-commits
+name: Rewrite messy local history to Angular Style on a temp branch
+discipline: rewrite-commits
+fixture:
+  files:
+    - path: setup.js
+      content: |
+        const { execSync } = require('child_process');
+        function commit(file, content, msg) {
+          require('fs').writeFileSync(file, content);
+          execSync(`git add ${file}`);
+          execSync(`git commit -qm "${msg}"`);
+        }
+        commit('a.txt', 'one', 'wip');
+        commit('b.txt', 'two', 'fixed thing');
+        commit('c.txt', 'three', 'more changes');
+        console.log('history seeded');
+    - path: package.json
+      content: |
+        { "name": "rewrite-commits-fixture" }
+prompt: "Run `node setup.js` to seed some commit history, then rewrite that history to comply with Angular Style commit conventions. Do this on a temporary branch — never rewrite main directly."
+# Still hit max_turns at 20 with hook overhead in the treatment workspace
+# (interactive-rebase-equivalent scripting is turn-hungry); bumped again but
+# this specific value is unverified by a live run - re-check before trusting
+# a treatment session-error on this case.
+max_turns: 28
+expectations:
+  - type: command_exit_0
+    command: "git branch --show-current | grep -qv -x main"
+    description: history was rewritten on a temp branch, not on main
+  - type: command_exit_0
+    command: 'git log --oneline | grep -qE "^[0-9a-f]+ (feat|fix|refactor|chore|docs|test|style|perf|build|ci)(\([^)]*\))?: "'
+    description: at least one rewritten commit follows Angular Style

package/behavioral-evals/cases/baseline-security-review.yaml

@@ -0,0 +1,28 @@
+# Baseline case: Security review
+id: baseline-security-review
+name: Security review of authentication code
+discipline: security-review
+fixture:
+  files:
+    - path: auth.js
+      content: |
+        const crypto = require('crypto');
+        
+        function hashPassword(password) {
+          return crypto.createHash('md5').update(password).digest('hex');
+        }
+        
+        function verifyPassword(password, hash) {
+          return hashPassword(password) === hash;
+        }
+        
+        module.exports = { hashPassword, verifyPassword };
+prompt: "Review auth.js for security vulnerabilities."
+max_turns: 16
+expectations:
+  - type: trace_contains
+    value: "md5"
+    description: agent identified MD5 as insecure
+  - type: trace_contains
+    value: "bcrypt"
+    description: agent suggested better hashing algorithm

package/behavioral-evals/cases/baseline-self-evolve.yaml

@@ -0,0 +1,31 @@
+# Baseline case: Self-evolve persists a root-cause rule after a resolved bug
+#
+# The script assertion uses structured completed-tool evidence, and the
+# file_contains check independently proves that the rule was persisted.
+id: baseline-self-evolve
+name: Persist a defensive rule after a resolved recurring bug
+discipline: self-evolve
+fixture:
+  files:
+    - path: NOTES.md
+      content: |
+        You just spent a while debugging a recurring bug in this project and
+        fixed it. Root cause: the build script cached file contents keyed by
+        filename only. When a file was renamed but the content stayed the
+        same, the cache key collided with a stale entry from before the
+        rename, so the old (pre-rename) content kept getting served instead
+        of a fresh read. Fix: key the cache by content hash instead of
+        filename.
+    - path: package.json
+      content: |
+        { "name": "self-evolve-fixture" }
+prompt: "Reflect on the root cause described in NOTES.md and persist a defensive rule for future sessions."
+max_turns: 14
+expectations:
+  - type: tool_completed
+    command: "persist-memory.js"
+    description: invoked self-evolve's persistence script rather than editing memory files ad hoc
+  - type: file_contains
+    path: "memories/repo/RULES.md"
+    value: "cache"
+    description: a rule about the cache root cause was persisted

package/behavioral-evals/cases/baseline-simple-bugfix.yaml

@@ -0,0 +1,33 @@
+# Baseline case: Simple bug fix with clear instructions
+id: baseline-simple-bugfix
+name: Simple bug fix with clear instructions
+discipline: tier1
+loaded_skills:
+  - harness-everything
+fixture:
+  files:
+    - path: math.js
+      content: |
+        function add(a, b) {
+          return a - b; // bug: should be a + b
+        }
+        module.exports = add;
+    - path: math.test.js
+      content: |
+        const add = require('./math');
+        const assert = require('assert');
+        assert.equal(add(2, 3), 5);
+        assert.equal(add(0, 0), 0);
+        console.log('all tests pass');
+    - path: package.json
+      content: |
+        { "name": "math-test", "scripts": { "test": "node math.test.js" } }
+prompt: "Fix the bug in math.js so the tests pass."
+max_turns: 12
+expectations:
+  - type: command_exit_0
+    command: "npm test"
+    description: tests pass after fix
+  - type: trace_contains
+    value: "add"
+    description: agent understood the function

package/behavioral-evals/cases/baseline-skill-creator.yaml

@@ -0,0 +1,20 @@
+# Baseline case: Skill-creator authors a SKILL.md against the Skill Contract format
+id: baseline-skill-creator
+name: New SKILL.md follows the mandatory Skill Contract format
+discipline: skill-creator
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "skill-creator-fixture" }
+prompt: "Create a new Harness skill called `changelog-writer` that drafts CHANGELOG entries from a git diff. Follow the Skill Contract format."
+max_turns: 14
+expectations:
+  - type: file_contains
+    path: ".claude/skills/changelog-writer/SKILL.md"
+    value: "Skill Contract"
+    description: new SKILL.md includes the mandatory Skill Contract table
+  - type: file_contains
+    path: ".claude/skills/changelog-writer/SKILL.md"
+    value: "USE FOR"
+    description: new SKILL.md includes the USE FOR section

package/behavioral-evals/cases/baseline-skill-style.yaml

@@ -0,0 +1,21 @@
+# Baseline case: Skill-style refactors a SKILL.md to the Harness writing standard
+id: baseline-skill-style
+name: Refactor a non-conforming SKILL.md to the house style
+discipline: skill-style
+fixture:
+  files:
+    - path: messy-skill/SKILL.md
+      content: |
+        This skill helps with stuff related to formatting text in various ways
+        depending on what the user wants, and it can be used whenever someone
+        needs some text formatted nicely, sort of like a general helper.
+    - path: package.json
+      content: |
+        { "name": "skill-style-fixture" }
+prompt: "Review and refactor messy-skill/SKILL.md to comply with the Harness skill style guide."
+max_turns: 12
+expectations:
+  - type: file_contains
+    path: "messy-skill/SKILL.md"
+    value: "Skill Contract"
+    description: refactored file includes the mandatory Skill Contract section

package/behavioral-evals/cases/baseline-tdd-workflow.yaml

@@ -0,0 +1,28 @@
+# Baseline case: TDD workflow
+id: baseline-tdd-workflow
+name: TDD workflow with test-first approach
+discipline: tdd
+fixture:
+  files:
+    - path: calculator.js
+      content: |
+        function add(a, b) { return a + b; }
+        function subtract(a, b) { return a - b; }
+        module.exports = { add, subtract };
+    - path: package.json
+      content: |
+        { "name": "calculator", "scripts": { "test": "node calculator.test.js" } }
+prompt: "Add multiply() and divide() functions using TDD — write tests first."
+max_turns: 20
+expectations:
+  - type: command_exit_0
+    command: "npm test"
+    description: all tests pass
+  - type: file_contains
+    path: "calculator.js"
+    value: "multiply"
+    description: multiply function exists
+  - type: file_contains
+    path: "calculator.js"
+    value: "divide"
+    description: divide function exists

package/behavioral-evals/cases/baseline-to-spec.yaml

@@ -0,0 +1,34 @@
+# Baseline case: To-spec's outline-preview gate holds even under a naive
+# pre-confirmation attempt
+#
+# First version of this case expected treatment to actually publish
+# docs/specs/*.md, given a prompt that pre-confirms ("I confirm the outline
+# — go ahead and publish"). Live run showed that's the wrong expectation:
+# to-spec's own Enforcement Gate is "outline preview before writing... write
+# only after user confirms" - a blanket pre-confirmation of an outline that
+# doesn't exist yet doesn't satisfy that gate, so treatment correctly showed
+# the outline and stopped without writing, ending the single headless turn.
+# Baseline (no skill) just wrote FEATURE_SPEC.md straight away, no outline
+# step. That contrast - the gate holding under a bypass attempt - is the
+# real, positive signal this case should check for, not file publication.
+id: baseline-to-spec
+name: Outline-preview gate fires even when the prompt tries to pre-confirm it
+discipline: to-spec
+fixture:
+  files:
+    - path: docs/specs/.gitkeep
+      content: |
+        placeholder
+    - path: PLAN.md
+      content: |
+        Agreed plan: add rate-limiting to the API using a token-bucket
+        algorithm, 100 requests/min per user.
+    - path: package.json
+      content: |
+        { "name": "to-spec-fixture" }
+prompt: "PLAN.md has the plan we already agreed on. I confirm the outline — go ahead and publish the feature spec for it now."
+max_turns: 14
+expectations:
+  - type: trace_contains
+    value: "Outline"
+    description: showed the mandatory outline preview instead of writing straight away on a naive pre-confirmation

package/behavioral-evals/cases/baseline-to-tickets.yaml

@@ -0,0 +1,24 @@
+# Baseline case: To-tickets slices a plan into blocking-edge-declared tickets
+#
+# .claude/tickets/ is an empirically observed write location from a live run
+# with no check-project-docs.js config present, not a hardcoded assumption
+# from the skill's own docs (its documented fallback is
+# .claude/harness-everything/tickets/). If a future run picks a different
+# fallback path, this expectation needs to move with it.
+id: baseline-to-tickets
+name: Break an approved plan into tracer-bullet tickets with blocking edges
+discipline: to-tickets
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "to-tickets-fixture" }
+prompt: "Break this plan into tracer-bullet tickets: (1) add input validation to the signup form, (2) send a welcome email after signup, where (2) depends on (1). I approve the granularity and edges as given — publish the tickets now."
+max_turns: 14
+expectations:
+  - type: command_exit_0
+    command: "grep -qil blocked .claude/tickets/*.md"
+    description: published tickets declare a blocking edge, checked in the ticket file itself rather than chat text
+  - type: command_exit_0
+    command: "grep -qil signup .claude/tickets/*.md"
+    description: tickets reference the actual work items from the plan

package/behavioral-evals/cases/baseline-todo-driven-workflow.yaml

@@ -0,0 +1,32 @@
+# Baseline case: Todo-driven workflow tracks one in-progress milestone at a time
+#
+# The todo assertion inspects the structured completed TodoWrite input. The
+# file_contains check independently proves the documentation milestone.
+id: baseline-todo-driven-workflow
+name: Track a multi-file task as todos with exactly one in-progress
+discipline: todo-driven-workflow
+fixture:
+  files:
+    - path: slug.js
+      content: |
+        module.exports = {};
+    - path: text.js
+      content: |
+        module.exports = {};
+    - path: README.md
+      content: |
+        # Utils
+    - path: package.json
+      content: |
+        { "name": "todo-driven-workflow-fixture" }
+prompt: "This is a multi-step task: add a slugify() function to slug.js, add a capitalize() function to text.js, and update README.md documenting both. Track this as todos, exactly one in-progress at a time."
+max_turns: 20
+expectations:
+  - type: tool_completed
+    name: TodoWrite
+    input_contains: in_progress
+    description: tracked milestones with an explicit in-progress state
+  - type: file_contains
+    path: "README.md"
+    value: "slugify"
+    description: documentation milestone was completed

package/behavioral-evals/cases/baseline-using-git-worktrees.yaml

@@ -0,0 +1,15 @@
+# Baseline case: Using-git-worktrees isolates feature work before editing
+id: baseline-using-git-worktrees
+name: Isolate feature work in a worktree before touching source
+discipline: using-git-worktrees
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "using-git-worktrees-fixture" }
+prompt: "Start a new feature: add a healthCheck() endpoint. Set up an isolated workspace for this before making any changes."
+max_turns: 14
+expectations:
+  - type: command_exit_0
+    command: "test -d .worktrees"
+    description: created an isolated .worktrees/<branch> workspace per the documented state mutation