harness-everything 0.3.6 → 0.4.1

package/behavioral-evals/results/2026-09-03-baseline-skill-style-pair.json

@@ -0,0 +1,133 @@
+{
+  "id": "baseline-skill-style",
+  "pair_id": "baseline-skill-style-1788477508512-a9b904",
+  "engine": "claude",
+  "model": "haiku",
+  "pressure": false,
+  "pressure_category": null,
+  "sample_unit": "paired case",
+  "arm_order": [
+    "baseline",
+    "treatment"
+  ],
+  "fixture_sha256": "f9ec5716326d8fcc888a4139d0df262555c6fbb89d68f5dd9c0391e428c30fc6",
+  "prompt_sha256": "fad3802938383f56a8e5e710f80083b13fe300e8a37b7ef3d71c9294837666f2",
+  "arms": {
+    "baseline": {
+      "arm": "baseline",
+      "pair_id": "baseline-skill-style-1788477508512-a9b904",
+      "harness_loaded": false,
+      "loaded_skills": [],
+      "attribution_required": false,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-skill-style-zMh4et",
+      "fixture_sha256": "f9ec5716326d8fcc888a4139d0df262555c6fbb89d68f5dd9c0391e428c30fc6",
+      "prompt_sha256": "fad3802938383f56a8e5e710f80083b13fe300e8a37b7ef3d71c9294837666f2",
+      "cost": 0.0422306,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-skill-style-zMh4et.transcript.json",
+      "usage": {
+        "input_tokens": 74,
+        "cache_creation_input_tokens": 6398,
+        "cache_read_input_tokens": 184696,
+        "output_tokens": 1975,
+        "output_tokens_details": {
+          "thinking_tokens": 815
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 6398,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 363,
+            "cache_read_input_tokens": 22018,
+            "cache_creation_input_tokens": 645,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 645
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 31289,
+      "num_turns": 10,
+      "expectations": [
+        {
+          "description": "refactored file includes the mandatory Skill Contract section",
+          "pass": false
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "Done! I've refactored the skill to comply with the Harness style guide. Key improvements:\n\n1. **Clear title** - Replaced vague prose with a focused title\n2. **Trigger conditions** - Added explicit guidance on when to invoke the skill\n3. **Structured instructions** - Replaced rambling text with numbered, actionable steps\n4. **Proper formatting** - Uses markdown headers correctly\n5. **Concise language** - Removed vague phrases like \"stuff,\" \"sort of,\" and \"nicely\"\n\nThe file now follows best practi"
+    },
+    "treatment": {
+      "arm": "treatment",
+      "pair_id": "baseline-skill-style-1788477508512-a9b904",
+      "harness_loaded": true,
+      "loaded_skills": [
+        "skill-style"
+      ],
+      "attribution_required": true,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-skill-style-slcHWc",
+      "fixture_sha256": "f9ec5716326d8fcc888a4139d0df262555c6fbb89d68f5dd9c0391e428c30fc6",
+      "prompt_sha256": "fad3802938383f56a8e5e710f80083b13fe300e8a37b7ef3d71c9294837666f2",
+      "cost": 0.044453400000000004,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-skill-style-slcHWc.transcript.json",
+      "usage": {
+        "input_tokens": 43,
+        "cache_creation_input_tokens": 11363,
+        "cache_read_input_tokens": 116144,
+        "output_tokens": 2014,
+        "output_tokens_details": {
+          "thinking_tokens": 678
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 11363,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 496,
+            "cache_read_input_tokens": 26454,
+            "cache_creation_input_tokens": 1174,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 1174
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 27562,
+      "num_turns": 6,
+      "expectations": [
+        {
+          "description": "refactored file includes the mandatory Skill Contract section",
+          "pass": true
+        }
+      ],
+      "outcome": "pass",
+      "tool_call_count": 0,
+      "trace_preview": "Done! I've refactored `messy-skill/SKILL.md` to comply with the Harness style guide. Here's what changed:\n\n**Added:**\n- **Frontmatter** with `name`, `description`, and `metadata` (type, triggers)\n- **Skill Contract table** defining Trigger/Input, Expected Output, State Mutations, and Enforcement Gate\n- **USE FOR / DO NOT USE FOR sections** to clarify scope boundaries\n- **Core Rules** with imperative language (\"MUST\", \"ALWAYS\") and script-driven enforcement references\n\n**Removed:**\n- Vague prose "
+    }
+  },
+  "verdict": "EFFECTIVE",
+  "date": "2026-09-03T23:19:32.908Z"
+}

package/behavioral-evals/results/2026-09-03-baseline-to-spec-pair.json

@@ -0,0 +1,133 @@
+{
+  "id": "baseline-to-spec",
+  "pair_id": "baseline-to-spec-1788478802174-6cf854",
+  "engine": "claude",
+  "model": "haiku",
+  "pressure": false,
+  "pressure_category": null,
+  "sample_unit": "paired case",
+  "arm_order": [
+    "treatment",
+    "baseline"
+  ],
+  "fixture_sha256": "eef5b2b56fe8bd3be3209e15936026fead1dedb249f3bb3689829543ca475ab9",
+  "prompt_sha256": "2e8ea53d16de67c7b905bf3076316dd42c8744605b04a959c5d344384835e339",
+  "arms": {
+    "treatment": {
+      "arm": "treatment",
+      "pair_id": "baseline-to-spec-1788478802174-6cf854",
+      "harness_loaded": true,
+      "loaded_skills": [
+        "to-spec"
+      ],
+      "attribution_required": true,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-spec-sC5vmt",
+      "fixture_sha256": "eef5b2b56fe8bd3be3209e15936026fead1dedb249f3bb3689829543ca475ab9",
+      "prompt_sha256": "2e8ea53d16de67c7b905bf3076316dd42c8744605b04a959c5d344384835e339",
+      "cost": 0.04517020000000001,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-spec-sC5vmt.transcript.json",
+      "usage": {
+        "input_tokens": 59,
+        "cache_creation_input_tokens": 10229,
+        "cache_read_input_tokens": 166782,
+        "output_tokens": 1595,
+        "output_tokens_details": {
+          "thinking_tokens": 666
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 10229,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 421,
+            "cache_read_input_tokens": 26317,
+            "cache_creation_input_tokens": 177,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 177
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 30059,
+      "num_turns": 8,
+      "expectations": [
+        {
+          "description": "showed the mandatory outline preview instead of writing straight away on a naive pre-confirmation",
+          "pass": true
+        }
+      ],
+      "outcome": "pass",
+      "tool_call_count": 0,
+      "trace_preview": "Perfect. I found `docs/specs/` as the target location. Here's the feature spec outline I'll create:\n\n**📋 Outline Preview: API Rate-Limiting Feature Spec**\n\n**Target:** `docs/specs/api-rate-limiting.md`  \n**Template:** feature-spec.md  \n**Status:** ready-for-agent\n\n```\n# API Rate-Limiting\n\n## Overview\nImplement rate-limiting on the API using a token-bucket algorithm to protect against abuse and ensure fair resource allocation across users.\n\n## Goal\nLimit API requests to 100 requests/min per user"
+    },
+    "baseline": {
+      "arm": "baseline",
+      "pair_id": "baseline-to-spec-1788478802174-6cf854",
+      "harness_loaded": false,
+      "loaded_skills": [],
+      "attribution_required": false,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-spec-3qZewH",
+      "fixture_sha256": "eef5b2b56fe8bd3be3209e15936026fead1dedb249f3bb3689829543ca475ab9",
+      "prompt_sha256": "2e8ea53d16de67c7b905bf3076316dd42c8744605b04a959c5d344384835e339",
+      "cost": 0.022660600000000003,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-spec-3qZewH.transcript.json",
+      "usage": {
+        "input_tokens": 26,
+        "cache_creation_input_tokens": 5122,
+        "cache_read_input_tokens": 56956,
+        "output_tokens": 1140,
+        "output_tokens_details": {
+          "thinking_tokens": 275
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 5122,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 161,
+            "cache_read_input_tokens": 20472,
+            "cache_creation_input_tokens": 915,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 915
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 14011,
+      "num_turns": 3,
+      "expectations": [
+        {
+          "description": "showed the mandatory outline preview instead of writing straight away on a naive pre-confirmation",
+          "pass": false
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "Done. I've created and published `FEATURE_SPEC.md` with the complete specification for the API rate-limiting feature using a token-bucket algorithm at 100 requests/minute per user. The spec includes functional/non-functional requirements, implementation details, acceptance criteria, and future considerations."
+    }
+  },
+  "verdict": "EFFECTIVE",
+  "date": "2026-09-03T23:40:52.501Z"
+}

package/behavioral-evals/results/2026-09-03-baseline-to-tickets-pair.json

@@ -0,0 +1,141 @@
+{
+  "id": "baseline-to-tickets",
+  "pair_id": "baseline-to-tickets-1788478735771-78595b",
+  "engine": "claude",
+  "model": "haiku",
+  "pressure": false,
+  "pressure_category": null,
+  "sample_unit": "paired case",
+  "arm_order": [
+    "treatment",
+    "baseline"
+  ],
+  "fixture_sha256": "ef406aa28eac1daf9b45c7086e412419fdb2dda269b00e1dd3511a2345c25f40",
+  "prompt_sha256": "6e8e15097f97cce6b88a08d3bc3bbcd07f50aa7a9acf74d26f6856896491b5ec",
+  "arms": {
+    "treatment": {
+      "arm": "treatment",
+      "pair_id": "baseline-to-tickets-1788478735771-78595b",
+      "harness_loaded": true,
+      "loaded_skills": [
+        "to-tickets"
+      ],
+      "attribution_required": true,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-tickets-61aGWj",
+      "fixture_sha256": "ef406aa28eac1daf9b45c7086e412419fdb2dda269b00e1dd3511a2345c25f40",
+      "prompt_sha256": "6e8e15097f97cce6b88a08d3bc3bbcd07f50aa7a9acf74d26f6856896491b5ec",
+      "cost": 0.05141370000000001,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-tickets-61aGWj.transcript.json",
+      "usage": {
+        "input_tokens": 59,
+        "cache_creation_input_tokens": 11585,
+        "cache_read_input_tokens": 169847,
+        "output_tokens": 2240,
+        "output_tokens_details": {
+          "thinking_tokens": 737
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 11585,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 209,
+            "cache_read_input_tokens": 26973,
+            "cache_creation_input_tokens": 877,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 877
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 42122,
+      "num_turns": 11,
+      "expectations": [
+        {
+          "description": "published tickets declare a blocking edge (checked in the ticket file itself, not chat text - the claude engine's trace only captures the final message, see #52)",
+          "pass": true
+        },
+        {
+          "description": "tickets reference the actual work items from the plan",
+          "pass": true
+        }
+      ],
+      "outcome": "pass",
+      "tool_call_count": 0,
+      "trace_preview": "✅ **Published 2 tickets to `.claude/tickets/`:**\n\n1. **`01-add-input-validation-signup-form.md`** — No dependencies\n   - Email, password, username validation (client & server)\n   - Real-time error feedback\n   - Ready for downstream work\n\n2. **`02-send-welcome-email-after-signup.md`** — Blocked by ticket 1\n   - Sends welcome email after successful signup\n   - Includes account confirmation and onboarding info\n   - Graceful error handling\n\nBoth tickets use cross-references (`[[01-...]]`, `[[02-...]"
+    },
+    "baseline": {
+      "arm": "baseline",
+      "pair_id": "baseline-to-tickets-1788478735771-78595b",
+      "harness_loaded": false,
+      "loaded_skills": [],
+      "attribution_required": false,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-tickets-dckXPF",
+      "fixture_sha256": "ef406aa28eac1daf9b45c7086e412419fdb2dda269b00e1dd3511a2345c25f40",
+      "prompt_sha256": "6e8e15097f97cce6b88a08d3bc3bbcd07f50aa7a9acf74d26f6856896491b5ec",
+      "cost": 0.037103500000000005,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-to-tickets-dckXPF.transcript.json",
+      "usage": {
+        "input_tokens": 64,
+        "cache_creation_input_tokens": 7567,
+        "cache_read_input_tokens": 149425,
+        "output_tokens": 1187,
+        "output_tokens_details": {
+          "thinking_tokens": 532
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 7567,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 121,
+            "cache_read_input_tokens": 23692,
+            "cache_creation_input_tokens": 140,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 140
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 17774,
+      "num_turns": 8,
+      "expectations": [
+        {
+          "description": "published tickets declare a blocking edge (checked in the ticket file itself, not chat text - the claude engine's trace only captures the final message, see #52)",
+          "pass": false
+        },
+        {
+          "description": "tickets reference the actual work items from the plan",
+          "pass": false
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "Done. Two tracer-bullet tickets published:\n\n- **#1** [pending] Add input validation to the signup form\n- **#2** [pending] Send a welcome email after signup — blocked by #1\n\nTask #2 won't be startable until #1 is completed."
+    }
+  },
+  "verdict": "EFFECTIVE",
+  "date": "2026-09-03T23:40:01.942Z"
+}

package/behavioral-evals/results/2026-09-03-baseline-todo-driven-workflow-pair.json

@@ -0,0 +1,141 @@
+{
+  "id": "baseline-todo-driven-workflow",
+  "pair_id": "baseline-todo-driven-workflow-1788478874892-d80d84",
+  "engine": "claude",
+  "model": "haiku",
+  "pressure": false,
+  "pressure_category": null,
+  "sample_unit": "paired case",
+  "arm_order": [
+    "baseline",
+    "treatment"
+  ],
+  "fixture_sha256": "95a3bc196ee2fe1ae8e7a3adf69061064b0469d05f2063123801889405accf1a",
+  "prompt_sha256": "dd7809e8b7c429943d6596910633a4f6e0426df87dd7e328a49bb72205727914",
+  "arms": {
+    "baseline": {
+      "arm": "baseline",
+      "pair_id": "baseline-todo-driven-workflow-1788478874892-d80d84",
+      "harness_loaded": false,
+      "loaded_skills": [],
+      "attribution_required": false,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-todo-driven-workflow-IbBcfI",
+      "fixture_sha256": "95a3bc196ee2fe1ae8e7a3adf69061064b0469d05f2063123801889405accf1a",
+      "prompt_sha256": "dd7809e8b7c429943d6596910633a4f6e0426df87dd7e328a49bb72205727914",
+      "cost": 0.06115589999999999,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-todo-driven-workflow-IbBcfI.transcript.json",
+      "usage": {
+        "input_tokens": 100,
+        "cache_creation_input_tokens": 9567,
+        "cache_read_input_tokens": 273279,
+        "output_tokens": 2715,
+        "output_tokens_details": {
+          "thinking_tokens": 669
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 9567,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 187,
+            "cache_read_input_tokens": 25688,
+            "cache_creation_input_tokens": 144,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 144
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 39856,
+      "num_turns": 19,
+      "expectations": [
+        {
+          "description": "tracked milestones with an explicit in-progress state",
+          "pass": false
+        },
+        {
+          "description": "documentation milestone was completed",
+          "pass": true
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "All three tasks completed! Here's what was done:\n\n1. ✅ **slug.js** - Added `slugify()` function that converts text to URL-friendly slugs (lowercase, hyphens for spaces, special chars removed)\n2. ✅ **text.js** - Added `capitalize()` function that capitalizes the first letter of text\n3. ✅ **README.md** - Added documentation section with function descriptions and usage examples\n\nAll functions are now exported and documented."
+    },
+    "treatment": {
+      "arm": "treatment",
+      "pair_id": "baseline-todo-driven-workflow-1788478874892-d80d84",
+      "harness_loaded": true,
+      "loaded_skills": [
+        "todo-driven-workflow"
+      ],
+      "attribution_required": true,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-todo-driven-workflow-QuaO3e",
+      "fixture_sha256": "95a3bc196ee2fe1ae8e7a3adf69061064b0469d05f2063123801889405accf1a",
+      "prompt_sha256": "dd7809e8b7c429943d6596910633a4f6e0426df87dd7e328a49bb72205727914",
+      "cost": 0.0744686,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-todo-driven-workflow-QuaO3e.transcript.json",
+      "usage": {
+        "input_tokens": 84,
+        "cache_creation_input_tokens": 15663,
+        "cache_read_input_tokens": 276086,
+        "output_tokens": 3090,
+        "output_tokens_details": {
+          "thinking_tokens": 950
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 15663,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 236,
+            "cache_read_input_tokens": 31561,
+            "cache_creation_input_tokens": 367,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 367
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 42065,
+      "num_turns": 14,
+      "expectations": [
+        {
+          "description": "tracked milestones with an explicit in-progress state",
+          "pass": false
+        },
+        {
+          "description": "documentation milestone was completed",
+          "pass": true
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "**All milestones complete!** \n\n✅ **Summary of changes:**\n\n1. **slug.js** — Added `slugify()` function that converts text to URL-friendly slugs (lowercase, space-to-hyphen, removes special characters)\n2. **text.js** — Added `capitalize()` function that capitalizes the first letter of text with empty string handling\n3. **README.md** — Documented both functions with clear descriptions and usage examples\n\nAll three sub-tasks tracked and executed sequentially, with verification at each step."
+    }
+  },
+  "verdict": "INEFFECTIVE",
+  "date": "2026-09-03T23:42:42.135Z"
+}