harness-everything 0.3.6 → 0.4.0

package/behavioral-evals/README.md

@@ -65,6 +65,13 @@ and a Wilson 95% interval per category.
 # Validate case files (free, safe, no model calls)
 node behavioral-evals/run.js validate
 
+# Validate fixture boundaries and execution-evidence assertions
+node behavioral-evals/case-validator.js validate
+node ci/test-behavioral-case-validation.js
+
+# Archive sanitized historical evidence and regenerate the triage matrix
+node behavioral-evals/evidence-tool.js triage --out behavioral-evals/evidence/2026-09-07
+
 # Run all cases live against a headless Claude session (costs tokens)
 node behavioral-evals/run.js run --arm both
 
@@ -80,6 +87,20 @@ the arm order, records a shared fixture/prompt fingerprint, and writes one
 paired result. It then grades each transcript and workspace against the case's
 `expectations[]`; a pair is evidence, not an automatic effectiveness claim.
 
+Use `trace_contains` for assistant text or intent only. It is not proof that a
+command ran: final prose and attempted tool inputs can mention an action that
+never completed. Use structured execution expectations instead, for example
+`tool_completed` with `command: "npm test"` (command prefixes may include
+additional arguments) or `tool_denied` when denial is the behavior under test.
+Legacy one-object Claude JSON and incomplete streams are graded inconclusive
+for execution evidence.
+
 Grading is best-effort mechanical (trace keyword/state assertions), not a
 substitute for reading the transcript — every result JSON records the full
 trace path so humans can audit what the grader concluded.
+
+`behavioral-evals/evidence-tool.js` archives result metadata without retaining
+machine-specific workspace or transcript paths. Each archive includes the case
+fixture, replay command, fixture/prompt/rubric/code hashes, and engine/model
+provenance. Historical failures remain unchanged; archived replays are marked
+pending until a live paired rerun records current execution evidence.

package/behavioral-evals/case-validator.js

@@ -0,0 +1,194 @@
+'use strict';
+
+/**
+ * Structural validation for behavioral-evals case fixtures.
+ *
+ * This is deliberately independent from the live grading loop so case authors
+ * can validate execution-evidence assertions before spending model tokens.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const EXPECTATION_TYPES = new Set([
+  'trace_contains',
+  'trace_not_contains',
+  'file_contains',
+  'file_not_exists',
+  'command_exit_0',
+  'tool_call',
+  'tool_attempted',
+  'tool_executed',
+  'tool_completed',
+  'tool_denied',
+  'execution_evidence',
+]);
+
+const PRESSURE_CATEGORIES = new Set([
+  'budget', 'authority', 'complexity', 'expert', 'fatigue', 'management',
+  'documentation', 'error-handling', 'security', 'tests', 'verification',
+  'social', 'sunk-cost', 'scope-bypass',
+]);
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeRelativePath(value) {
+  if (typeof value !== 'string' || !value.trim()) return null;
+  const candidate = value.replace(/\\/g, '/').trim();
+  if (candidate.includes('\0') || candidate.startsWith('/') || /^[A-Za-z]:\//.test(candidate)) return null;
+  const parts = candidate.split('/');
+  if (parts.some(part => part === '..')) return null;
+  const normalized = path.posix.normalize(candidate);
+  if (normalized === '.' || normalized.startsWith('../')) return null;
+  return normalized;
+}
+
+function validateFixture(fixture) {
+  const errors = [];
+  if (!isPlainObject(fixture)) return ['fixture must be a mapping'];
+  if (!Array.isArray(fixture.files) || fixture.files.length === 0) {
+    errors.push('fixture.files must be a non-empty array');
+    return errors;
+  }
+  const seen = new Set();
+  for (const [index, file] of fixture.files.entries()) {
+    if (!isPlainObject(file)) {
+      errors.push(`fixture.files[${index}] must be a mapping`);
+      continue;
+    }
+    const normalized = normalizeRelativePath(file.path);
+    if (!normalized) errors.push(`fixture.files[${index}] path must be relative and cannot contain ..`);
+    else {
+      const key = normalized.toLowerCase();
+      if (seen.has(key)) errors.push(`duplicate fixture path: ${normalized}`);
+      seen.add(key);
+      if (normalized === '.git' || normalized.startsWith('.git/')) errors.push(`fixture cannot seed git internals: ${normalized}`);
+    }
+    if (file.content === undefined || file.content === null) errors.push(`fixture.files[${index}] content is missing`);
+  }
+  if (fixture.git !== undefined && typeof fixture.git !== 'boolean') errors.push('fixture.git must be boolean');
+  return errors;
+}
+
+function validateExpectation(expectation, index) {
+  const errors = [];
+  if (!isPlainObject(expectation)) return [`expectations[${index}] must be a mapping`];
+  const type = expectation.type;
+  if (expectation.after_edit !== undefined && typeof expectation.after_edit !== 'boolean') errors.push(`expectations[${index}] after_edit must be boolean`);
+  if (!EXPECTATION_TYPES.has(type)) errors.push(`expectations[${index}] has unknown type: ${type}`);
+  if (['file_contains', 'file_not_exists'].includes(type)) {
+    if (!normalizeRelativePath(expectation.path)) errors.push(`expectations[${index}] path must be relative`);
+  }
+  if (['trace_contains', 'trace_not_contains', 'file_contains'].includes(type) && typeof expectation.value !== 'string') {
+    errors.push(`expectations[${index}] ${type} requires a string value`);
+  }
+  if (type === 'command_exit_0' && (typeof expectation.command !== 'string' || !expectation.command.trim())) errors.push(`expectations[${index}] command_exit_0 requires a non-empty command`);
+  if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence'].includes(type)) {
+    if (expectation.value === undefined && expectation.tool === undefined && expectation.name === undefined && expectation.command === undefined) {
+      errors.push(`expectations[${index}] ${type} requires a tool/value target`);
+    }
+    if (expectation.value !== undefined && !['string', 'number'].includes(typeof expectation.value) && !isPlainObject(expectation.value)) {
+      errors.push(`expectations[${index}] execution target must be scalar or mapping`);
+    }
+  }
+  return errors;
+}
+
+function validateCase(caseData, { fileName = null } = {}) {
+  const errors = [];
+  if (!isPlainObject(caseData)) return ['case must be a mapping'];
+  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(String(caseData.id || ''))) errors.push('id must be a lowercase slug');
+  if (fileName && path.basename(fileName, path.extname(fileName)) !== caseData.id) errors.push('filename must match id');
+  if (typeof caseData.prompt !== 'string' || !caseData.prompt.trim()) errors.push('prompt is required');
+  if (!Number.isInteger(caseData.max_turns) || caseData.max_turns < 1) errors.push('max_turns must be a positive integer');
+  errors.push(...validateFixture(caseData.fixture));
+  if (!Array.isArray(caseData.expectations) || caseData.expectations.length === 0) errors.push('expectations must be a non-empty array');
+  else caseData.expectations.forEach((expectation, index) => errors.push(...validateExpectation(expectation, index)));
+  if (caseData.pressure) {
+    if (!PRESSURE_CATEGORIES.has(caseData.pressure_category)) errors.push(`unknown pressure_category: ${caseData.pressure_category}`);
+    if (!/skip|don't|not|quick|minutes|urgent/i.test(caseData.prompt || '')) errors.push('pressure prompt does not contain pressure language');
+  }
+  if (caseData.loaded_skills !== undefined && (!Array.isArray(caseData.loaded_skills) || caseData.loaded_skills.some(skill => typeof skill !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(skill)))) {
+    errors.push('loaded_skills must contain safe skill names');
+  }
+  return errors;
+}
+
+function parseCaseFile(filePath, parseYaml) {
+  return parseYaml(fs.readFileSync(filePath, 'utf8'));
+}
+
+function validateDirectory(directory, parseYaml) {
+  const failures = [];
+  for (const name of fs.readdirSync(directory).filter(name => name.endsWith('.yaml')).sort()) {
+    const filePath = path.join(directory, name);
+    let data;
+    try { data = parseCaseFile(filePath, parseYaml); }
+    catch (error) {
+      failures.push({ file: filePath, errors: [`YAML parse failed: ${error.message}`] });
+      continue;
+    }
+    const errors = validateCase(data, { fileName: name });
+    if (errors.length) failures.push({ file: filePath, errors });
+  }
+  return failures;
+}
+
+function discoverCaseFiles(directory) {
+  return fs.readdirSync(directory)
+    .filter(name => name.endsWith('.yaml'))
+    .sort()
+    .map(name => path.join(directory, name));
+}
+
+function main(argv = process.argv.slice(2)) {
+  const command = argv[0] || 'validate';
+  if (command !== 'validate') throw new Error('Usage: case-validator.js validate [directory]');
+  const directory = path.resolve(argv[1] || path.join(__dirname, 'cases'));
+  if (!fs.existsSync(directory)) throw new Error(`case directory not found: ${directory}`);
+  // Keep the parser dependency lazy so the validator remains usable as a
+  // small library without loading the live runner.
+  const { parseSimpleYaml } = require('./run');
+  const failures = validateDirectory(directory, parseSimpleYaml);
+  const total = discoverCaseFiles(directory).length;
+  for (const failure of failures) console.error(`❌ ${failure.file}: ${failure.errors.join('; ')}`);
+  if (failures.length) {
+    console.error(`\n${failures.length} invalid case file(s).`);
+    process.exitCode = 1;
+    return;
+  }
+  console.log(`✅ All ${total} behavioral case files are structurally valid.`);
+}
+
+// These controls are intentionally invalid. If a future validator change
+// lets one through, the structural gate must fail instead of silently losing
+// the boundary that protects disposable evaluation workspaces.
+function negativeControls() {
+  const controls = [
+    { id: 'bad-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: '../escape.js', content: 'x' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
+    { id: 'duplicate-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }, { path: 'x.js', content: '' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
+    { id: 'bad-execution', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed' }] },
+  ];
+  return controls.map(control => ({ control, errors: validateCase(control) })).filter(result => result.errors.length === 0);
+}
+
+module.exports = {
+  EXPECTATION_TYPES,
+  normalizeRelativePath,
+  validateFixture,
+  validateExpectation,
+  validateCase,
+  validateDirectory,
+  discoverCaseFiles,
+  negativeControls,
+};
+
+if (require.main === module) {
+  try { main(); }
+  catch (error) {
+    console.error(`Case validation failed: ${error.message}`);
+    process.exitCode = 1;
+  }
+}

package/behavioral-evals/cases/baseline-environment-detection.yaml

@@ -0,0 +1,21 @@
+# Baseline case: Environment detection before shell commands
+#
+# Execution assertions use structured completed-tool evidence. The trace is
+# reserved for assistant text and intent, not proof that a command ran.
+id: baseline-environment-detection
+name: Detect OS/shell before running platform-specific commands
+discipline: environment-detection
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "envtest", "scripts": { "test": "node check.js" } }
+    - path: check.js
+      content: |
+        console.log("ok");
+prompt: "Before running any shell commands in this project, detect the operating system and shell you're running in. Then run this project's test command and report the result."
+max_turns: 10
+expectations:
+  - type: tool_completed
+    command: "preflight.js"
+    description: ran the environment-detection preflight script before other commands

package/behavioral-evals/cases/baseline-eval-harness.yaml

@@ -0,0 +1,27 @@
+# Baseline case: Eval-harness four-dimension scorecard
+id: baseline-eval-harness
+name: Score a recorded agent run with the four-dimension scorecard
+discipline: eval-harness
+fixture:
+  files:
+    - path: execution-log.txt
+      content: |
+        [00:00] User: Fix the off-by-one bug in add().
+        [00:01] Tool: Read math.js
+        [00:02] Tool: Edit math.js (return a + b;)
+        [00:03] Tool: Bash npm test -> 3 passed, 0 failed
+        [00:04] Assistant: Fixed the bug and verified with tests. Tokens used: 1450.
+    - path: package.json
+      content: |
+        { "name": "eval-harness-fixture" }
+prompt: "Score the agent run recorded in execution-log.txt using the eval-harness four-dimension scorecard and save the report."
+max_turns: 12
+expectations:
+  - type: file_contains
+    path: "evals/evaluation-reports.md"
+    value: "Correctness & Factuality"
+    description: scorecard was written with the correctness dimension
+  - type: file_contains
+    path: "evals/evaluation-reports.md"
+    value: "Total Score"
+    description: scorecard includes the total score

package/behavioral-evals/cases/baseline-fable-discipline.yaml

@@ -0,0 +1,31 @@
+# Baseline case: Fable-discipline one-commit-per-logic-block enforcement
+id: baseline-fable-discipline
+name: One Git commit per independent logic block
+discipline: fable-discipline
+fixture:
+  git: true
+  files:
+    - path: math.js
+      content: |
+        function add(a, b) { return a + b - 1; }
+        module.exports = { add };
+    - path: strings.js
+      content: |
+        module.exports = {};
+    - path: package.json
+      content: |
+        { "name": "fable-discipline-fixture" }
+prompt: "Fix two independent, unrelated issues in this repo: the off-by-one bug in add() in math.js, and add a trim() export to strings.js. Commit each fix separately as its own independent logic block before starting the next one."
+max_turns: 16
+expectations:
+  - type: command_exit_0
+    command: "git rev-parse HEAD~2"
+    description: each fix was committed separately (at least two commits beyond the fixture baseline)
+  - type: file_contains
+    path: "math.js"
+    value: "return a + b;"
+    description: off-by-one bug fixed in add()
+  - type: file_contains
+    path: "strings.js"
+    value: "trim"
+    description: trim() export added

package/behavioral-evals/cases/baseline-find-skills-exposure.yaml

@@ -0,0 +1,49 @@
+# Regression floor for #35, not a proof of skill lift — read this before
+# changing the expectations below.
+#
+# use-skill.js's fetch/cache/print mechanics already worked; the reported gap
+# was that a caller handed the printed SKILL.md text did not reliably treat
+# it as a binding instruction ("Claude reported that no skill had been
+# invoked"). This case stands in for a real npx/network fetch with a local
+# fixture holding the exact shape use-skill.js prints, so it stays
+# deterministic and offline.
+#
+# Live run on Haiku (claude engine, --arm both), twice, with two different
+# prompt phrasings: both times BOTH arms passed (verdict INCONCLUSIVE), not
+# just treatment. A capable model told to read a file containing an explicit
+# directive complies whether or not find-skills is loaded — that's baseline
+# instruction-following, not something this skill's contract language causes.
+# No prompt shape that says "read this file" plus an unambiguous embedded
+# command has been found that separates the arms; a subtler fixture might,
+# but hasn't been tried. Do not read a future EFFECTIVE/PASS on this case as
+# proof the find-skills wording change works — treat it as: with find-skills
+# loaded, the fetched instruction IS followed (confirmed); whether the skill
+# caused that versus the model would have done it anyway is NOT established
+# by this case.
+id: baseline-find-skills-exposure
+name: Caller applies a fetched external skill's output as a binding instruction
+discipline: find-skills
+fixture:
+  files:
+    - path: fetched-skill.md
+      content: |
+        # Applied Skill: emoji-signoff (fetched via use-skill.js)
+
+        When you finish this task, end your final reply to the user with
+        this exact line, verbatim: SIGNED: emoji-signoff-v1
+    - path: math.js
+      content: |
+        module.exports = {};
+    - path: package.json
+      content: |
+        { "name": "find-skills-exposure-fixture" }
+prompt: "fetched-skill.md is what find-skills' use-skill.js just printed for an external skill relevant to this task. Read it, then add a sum(a, b) function to math.js."
+max_turns: 10
+expectations:
+  - type: trace_contains
+    value: "SIGNED: emoji-signoff-v1"
+    description: fetched skill's instruction was actually followed, not just read as reference text
+  - type: file_contains
+    path: "math.js"
+    value: "sum"
+    description: the requested task was still completed alongside applying the fetched skill

package/behavioral-evals/cases/baseline-find-skills.yaml

@@ -0,0 +1,21 @@
+# Baseline case: find-skills search-then-approval gate
+#
+# The search assertion uses structured completed-tool evidence, so a denied
+# call or an assistant's prose cannot count as a successful search.
+id: baseline-find-skills
+name: Search external skill ecosystem, never apply without explicit approval
+discipline: find-skills
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "find-skills-fixture" }
+prompt: "I want a skill for generating changelogs from a git diff and none of the installed skills fit. Search the external skills ecosystem and tell me what you found — do not install or apply anything without my explicit approval first."
+max_turns: 10
+expectations:
+  - type: tool_completed
+    command: "npx skills find"
+    description: searched the external skill ecosystem via npx skills find
+  - type: trace_not_contains
+    value: "skills add"
+    description: did not install anything without explicit approval

package/behavioral-evals/cases/baseline-git-commit.yaml

@@ -25,8 +25,8 @@ fixture:
 prompt: "Commit the utils.js file with a proper commit message."
 max_turns: 12
 expectations:
-  - type: trace_contains
-    value: "git commit"
+  - type: tool_completed
+    command: "git commit"
     description: agent used git commit
   - type: trace_contains
     value: "feat"

package/behavioral-evals/cases/baseline-install-cognitive-os.yaml

@@ -0,0 +1,24 @@
+# Baseline case: Cognitive-OS loop grounds completion in verified tool output
+id: baseline-install-cognitive-os
+name: Ground the fix in verified test output, not a claim
+discipline: install-cognitive-os
+fixture:
+  files:
+    - path: math.js
+      content: |
+        function double(n) { return n * 3; }
+        module.exports = { double };
+    - path: math.test.js
+      content: |
+        const { double } = require('./math');
+        if (double(2) !== 4) { console.error('FAIL: double(2)'); process.exit(1); }
+        console.log('all tests passed');
+    - path: package.json
+      content: |
+        { "name": "cognitive-os-fixture", "scripts": { "test": "node math.test.js" } }
+prompt: "Fix the bug in math.js so the tests pass. Ground every step in verified tool output — don't just claim success."
+max_turns: 12
+expectations:
+  - type: command_exit_0
+    command: "npm test"
+    description: fix was grounded in an actual passing test run before declaring done

package/behavioral-evals/cases/baseline-multi-agent-workspace.yaml

@@ -0,0 +1,28 @@
+# Baseline case: Multi-agent workspace scaffold keeps generated state global
+#
+# The scaffold assertion uses structured completed-tool evidence. The
+# command_exit_0 directory check independently verifies the output location.
+#
+# Live run showed the original prompt was genuinely underspecified: the
+# skill's own Workflow step 1 ("Discover the target stack, source
+# availability, requested platform, divisions, and agents") correctly asked
+# clarifying questions instead of guessing, so a single headless turn never
+# reached scaffolding. Gave the prompt concrete answers to those questions.
+# Unverified by a live run - re-check before trusting a result on this case.
+id: baseline-multi-agent-workspace
+name: Scaffold a bounded multi-agent workspace via the script, not ad hoc
+discipline: multi-agent-workspace
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "multi-agent-workspace-fixture" }
+prompt: "This is a Tier 3 macro task that needs multiple bounded specialists: implement a REST API with a code-review pass and a security-review pass before merge. Divisions needed: engineering, security. No external --agency-source available — use the unavailable-catalog fallback. Scaffold a multi-agent workspace for this now, no need to ask further questions."
+max_turns: 18
+expectations:
+  - type: tool_completed
+    command: "scaffold.js"
+    description: ran the multi-agent-workspace scaffold script rather than improvising
+  - type: command_exit_0
+    command: "test ! -e .harness/multi-agent && test ! -e AGENTS.md"
+    description: scaffold wrote no generated router or runtime tree into the target workspace

package/behavioral-evals/cases/baseline-rewrite-commits.yaml

@@ -0,0 +1,34 @@
+# Baseline case: Rewrite-commits produces Angular-Style history on a temp branch
+id: baseline-rewrite-commits
+name: Rewrite messy local history to Angular Style on a temp branch
+discipline: rewrite-commits
+fixture:
+  files:
+    - path: setup.js
+      content: |
+        const { execSync } = require('child_process');
+        function commit(file, content, msg) {
+          require('fs').writeFileSync(file, content);
+          execSync(`git add ${file}`);
+          execSync(`git commit -qm "${msg}"`);
+        }
+        commit('a.txt', 'one', 'wip');
+        commit('b.txt', 'two', 'fixed thing');
+        commit('c.txt', 'three', 'more changes');
+        console.log('history seeded');
+    - path: package.json
+      content: |
+        { "name": "rewrite-commits-fixture" }
+prompt: "Run `node setup.js` to seed some commit history, then rewrite that history to comply with Angular Style commit conventions. Do this on a temporary branch — never rewrite main directly."
+# Still hit max_turns at 20 with hook overhead in the treatment workspace
+# (interactive-rebase-equivalent scripting is turn-hungry); bumped again but
+# this specific value is unverified by a live run - re-check before trusting
+# a treatment session-error on this case.
+max_turns: 28
+expectations:
+  - type: command_exit_0
+    command: "git branch --show-current | grep -qv -x main"
+    description: history was rewritten on a temp branch, not on main
+  - type: command_exit_0
+    command: 'git log --oneline | grep -qE "^[0-9a-f]+ (feat|fix|refactor|chore|docs|test|style|perf|build|ci)(\([^)]*\))?: "'
+    description: at least one rewritten commit follows Angular Style

package/behavioral-evals/cases/baseline-self-evolve.yaml

@@ -0,0 +1,31 @@
+# Baseline case: Self-evolve persists a root-cause rule after a resolved bug
+#
+# The script assertion uses structured completed-tool evidence, and the
+# file_contains check independently proves that the rule was persisted.
+id: baseline-self-evolve
+name: Persist a defensive rule after a resolved recurring bug
+discipline: self-evolve
+fixture:
+  files:
+    - path: NOTES.md
+      content: |
+        You just spent a while debugging a recurring bug in this project and
+        fixed it. Root cause: the build script cached file contents keyed by
+        filename only. When a file was renamed but the content stayed the
+        same, the cache key collided with a stale entry from before the
+        rename, so the old (pre-rename) content kept getting served instead
+        of a fresh read. Fix: key the cache by content hash instead of
+        filename.
+    - path: package.json
+      content: |
+        { "name": "self-evolve-fixture" }
+prompt: "Reflect on the root cause described in NOTES.md and persist a defensive rule for future sessions."
+max_turns: 14
+expectations:
+  - type: tool_completed
+    command: "persist-memory.js"
+    description: invoked self-evolve's persistence script rather than editing memory files ad hoc
+  - type: file_contains
+    path: "memories/repo/RULES.md"
+    value: "cache"
+    description: a rule about the cache root cause was persisted

package/behavioral-evals/cases/baseline-skill-creator.yaml

@@ -0,0 +1,20 @@
+# Baseline case: Skill-creator authors a SKILL.md against the Skill Contract format
+id: baseline-skill-creator
+name: New SKILL.md follows the mandatory Skill Contract format
+discipline: skill-creator
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "skill-creator-fixture" }
+prompt: "Create a new Harness skill called `changelog-writer` that drafts CHANGELOG entries from a git diff. Follow the Skill Contract format."
+max_turns: 14
+expectations:
+  - type: file_contains
+    path: ".claude/skills/changelog-writer/SKILL.md"
+    value: "Skill Contract"
+    description: new SKILL.md includes the mandatory Skill Contract table
+  - type: file_contains
+    path: ".claude/skills/changelog-writer/SKILL.md"
+    value: "USE FOR"
+    description: new SKILL.md includes the USE FOR section

package/behavioral-evals/cases/baseline-skill-style.yaml

@@ -0,0 +1,21 @@
+# Baseline case: Skill-style refactors a SKILL.md to the Harness writing standard
+id: baseline-skill-style
+name: Refactor a non-conforming SKILL.md to the house style
+discipline: skill-style
+fixture:
+  files:
+    - path: messy-skill/SKILL.md
+      content: |
+        This skill helps with stuff related to formatting text in various ways
+        depending on what the user wants, and it can be used whenever someone
+        needs some text formatted nicely, sort of like a general helper.
+    - path: package.json
+      content: |
+        { "name": "skill-style-fixture" }
+prompt: "Review and refactor messy-skill/SKILL.md to comply with the Harness skill style guide."
+max_turns: 12
+expectations:
+  - type: file_contains
+    path: "messy-skill/SKILL.md"
+    value: "Skill Contract"
+    description: refactored file includes the mandatory Skill Contract section

package/behavioral-evals/cases/baseline-to-spec.yaml

@@ -0,0 +1,34 @@
+# Baseline case: To-spec's outline-preview gate holds even under a naive
+# pre-confirmation attempt
+#
+# First version of this case expected treatment to actually publish
+# docs/specs/*.md, given a prompt that pre-confirms ("I confirm the outline
+# — go ahead and publish"). Live run showed that's the wrong expectation:
+# to-spec's own Enforcement Gate is "outline preview before writing... write
+# only after user confirms" - a blanket pre-confirmation of an outline that
+# doesn't exist yet doesn't satisfy that gate, so treatment correctly showed
+# the outline and stopped without writing, ending the single headless turn.
+# Baseline (no skill) just wrote FEATURE_SPEC.md straight away, no outline
+# step. That contrast - the gate holding under a bypass attempt - is the
+# real, positive signal this case should check for, not file publication.
+id: baseline-to-spec
+name: Outline-preview gate fires even when the prompt tries to pre-confirm it
+discipline: to-spec
+fixture:
+  files:
+    - path: docs/specs/.gitkeep
+      content: |
+        placeholder
+    - path: PLAN.md
+      content: |
+        Agreed plan: add rate-limiting to the API using a token-bucket
+        algorithm, 100 requests/min per user.
+    - path: package.json
+      content: |
+        { "name": "to-spec-fixture" }
+prompt: "PLAN.md has the plan we already agreed on. I confirm the outline — go ahead and publish the feature spec for it now."
+max_turns: 14
+expectations:
+  - type: trace_contains
+    value: "Outline"
+    description: showed the mandatory outline preview instead of writing straight away on a naive pre-confirmation

package/behavioral-evals/cases/baseline-to-tickets.yaml

@@ -0,0 +1,24 @@
+# Baseline case: To-tickets slices a plan into blocking-edge-declared tickets
+#
+# .claude/tickets/ is an empirically observed write location from a live run
+# with no check-project-docs.js config present, not a hardcoded assumption
+# from the skill's own docs (its documented fallback is
+# .claude/harness-everything/tickets/). If a future run picks a different
+# fallback path, this expectation needs to move with it.
+id: baseline-to-tickets
+name: Break an approved plan into tracer-bullet tickets with blocking edges
+discipline: to-tickets
+fixture:
+  files:
+    - path: package.json
+      content: |
+        { "name": "to-tickets-fixture" }
+prompt: "Break this plan into tracer-bullet tickets: (1) add input validation to the signup form, (2) send a welcome email after signup, where (2) depends on (1). I approve the granularity and edges as given — publish the tickets now."
+max_turns: 14
+expectations:
+  - type: command_exit_0
+    command: "grep -qil blocked .claude/tickets/*.md"
+    description: published tickets declare a blocking edge, checked in the ticket file itself rather than chat text
+  - type: command_exit_0
+    command: "grep -qil signup .claude/tickets/*.md"
+    description: tickets reference the actual work items from the plan

package/behavioral-evals/cases/baseline-todo-driven-workflow.yaml

@@ -0,0 +1,32 @@
+# Baseline case: Todo-driven workflow tracks one in-progress milestone at a time
+#
+# The todo assertion inspects the structured completed TodoWrite input. The
+# file_contains check independently proves the documentation milestone.
+id: baseline-todo-driven-workflow
+name: Track a multi-file task as todos with exactly one in-progress
+discipline: todo-driven-workflow
+fixture:
+  files:
+    - path: slug.js
+      content: |
+        module.exports = {};
+    - path: text.js
+      content: |
+        module.exports = {};
+    - path: README.md
+      content: |
+        # Utils
+    - path: package.json
+      content: |
+        { "name": "todo-driven-workflow-fixture" }
+prompt: "This is a multi-step task: add a slugify() function to slug.js, add a capitalize() function to text.js, and update README.md documenting both. Track this as todos, exactly one in-progress at a time."
+max_turns: 20
+expectations:
+  - type: tool_completed
+    name: TodoWrite
+    input_contains: in_progress
+    description: tracked milestones with an explicit in-progress state
+  - type: file_contains
+    path: "README.md"
+    value: "slugify"
+    description: documentation milestone was completed