@data-matters.io/agent-harness 0.1.2 → 0.1.4

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@data-matters.io/agent-harness",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Spec-Driven Development with Claude Code: gates, task loop and standards installed into a product repository",
   "license": "UNLICENSED",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/data-matters-io/data-matters-agent-harness.git"
+    "url": "git+https://github.com/data-matters-io/agent-harness.git"
   },
   "bin": {
     "harness": "bin/harness"

package/templates/.claude/settings.json

@@ -43,6 +43,7 @@
       "Edit(./.harness/pr-scope.sh)",
       "Edit(./.harness/deps.sh)",
       "Edit(./.harness/db-smoke.sh)",
+      "Edit(./.harness/app-smoke.sh)",
       "Edit(./.harness/models.conf)",
       "Edit(./.harness/models.local.conf)",
       "Edit(./.harness/model-flags.sh)",

package/templates/.harness/app-smoke.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Application smoke gate (ADR 0024): builds the app, starts it against a migrated
+# and seeded database, runs the project's end-to-end smoke test, stops the app.
+# Meant for GATE_FINAL in gates.conf: it is slow (build + server + database), so
+# it runs once at the end of a change, not on every task or in the Stop hook.
+#
+# Usage: .harness/app-smoke.sh --url <url> --start <cmd> --test <cmd> [options]
+#   --url <url>       where the app answers; any HTTP response counts as up
+#   --start <cmd>     starts the server (runs in the background, in its own process group)
+#   --test <cmd>      the smoke test; its exit code is the gate's
+#   --build <cmd>     run first (e.g. the production build)
+#   --migrate <cmd> --seed <cmd>
+#                     prepare a throwaway database through .harness/db-smoke.sh
+#                     (ADR 0016); the commands get DATABASE_URL. Without them the
+#                     app runs against whatever the environment configures
+#   --if-exists <f>   pass without running anything when <f> does not exist, so a
+#                     preset can carry the gate for projects that may not have the test
+#   --wait <seconds>  how long to wait for the app (default 120)
+#
+# Example (gates.conf, nextjs):
+#   GATE_FINAL=("smoke:.harness/app-smoke.sh --if-exists scripts/smoke-test.ts \
+#     --migrate '$bin/prisma migrate deploy' --seed '$bin/prisma db seed' \
+#     --build '$bin/next build' --start '$bin/next start -p 3100' \
+#     --url http://127.0.0.1:3100 --test '$bin/tsx scripts/smoke-test.ts'")
+#
+# The server log goes to .harness/state/gates/app-smoke-server.log and its tail is
+# printed when the app does not come up or the test fails. A machine that cannot
+# run it skips the gate with HARNESS_SKIP_GATES=smoke (CI still can).
+set -euo pipefail
+
+usage() { sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }
+die() { echo "app-smoke: $*" >&2; exit 1; }
+
+url="" start="" test_cmd="" build="" migrate="" seed="" if_exists="" wait_s=120 inner=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --url) url="${2:?}"; shift 2 ;;
+    --start) start="${2:?}"; shift 2 ;;
+    --test) test_cmd="${2:?}"; shift 2 ;;
+    --build) build="${2:?}"; shift 2 ;;
+    --migrate) migrate="${2:?}"; shift 2 ;;
+    --seed) seed="${2:?}"; shift 2 ;;
+    --if-exists) if_exists="${2:?}"; shift 2 ;;
+    --wait) wait_s="${2:?}"; shift 2 ;;
+    --inner) inner=1; shift ;;
+    -h|--help) usage 0 ;;
+    *) die "unknown option: $1" ;;
+  esac
+done
+[[ -n "$url" && -n "$start" && -n "$test_cmd" ]] || usage 1
+[[ "$wait_s" =~ ^[0-9]+$ ]] || die "--wait needs a number of seconds"
+[[ "$url" =~ ^https?://([^/:]+):([0-9]+)(/.*)?$ ]] || die "--url needs http(s)://host:port"
+host="${BASH_REMATCH[1]}" port="${BASH_REMATCH[2]}"
+
+if [[ -n "$if_exists" && ! -e "$if_exists" ]]; then
+  echo "app-smoke: no $if_exists in this project; nothing to run"
+  exit 0
+fi
+
+# With a database to prepare, hand over to db-smoke.sh, which keeps its container
+# alive while this script runs again (--inner) as its check command.
+if [[ $inner -eq 0 && ( -n "$migrate" || -n "$seed" ) ]]; then
+  [[ -n "$migrate" && -n "$seed" ]] || die "--migrate and --seed go together"
+  inner_cmd="$(printf '%q ' "$0" --inner --url "$url" --start "$start" --test "$test_cmd" --wait "$wait_s")"
+  [[ -z "$build" ]] || inner_cmd+="$(printf -- "--build %q" "$build")"
+  exec "$(dirname "$0")/db-smoke.sh" "$migrate" "$seed" "$inner_cmd"
+fi
+
+log_dir="${HARNESS_STATE_DIR:-.harness/state}/gates"
+mkdir -p "$log_dir"
+server_log="$log_dir/app-smoke-server.log"
+server_pid=""
+stop_server() {
+  [[ -n "$server_pid" ]] || return 0
+  kill -TERM -- "-$server_pid" 2>/dev/null || true
+  sleep 1
+  kill -KILL -- "-$server_pid" 2>/dev/null || true
+}
+trap stop_server EXIT
+show_server_log() { echo "----- last 40 lines of the server log -----"; tail -n 40 "$server_log" 2>/dev/null || true; echo "-----"; }
+
+export PATH="$PWD/node_modules/.bin:$PATH"
+if [[ -n "$build" ]]; then
+  echo "app-smoke: build: $build"
+  bash -c "$build" || die "build failed"
+fi
+
+# The port must be free: a leftover server would answer instead of this build.
+if (exec 3<>"/dev/tcp/$host/$port") 2>/dev/null; then die "$host:$port is already in use"; fi
+echo "app-smoke: start: $start"
+setsid bash -c "$start" >"$server_log" 2>&1 &
+server_pid=$!
+
+up=0
+for (( i = 0; i < wait_s; i++ )); do
+  kill -0 "$server_pid" 2>/dev/null || { show_server_log; die "the app exited before answering on $url"; }
+  if (exec 3<>"/dev/tcp/$host/$port") 2>/dev/null; then up=1; break; fi
+  sleep 1
+done
+[[ $up -eq 1 ]] || { show_server_log; die "the app did not answer on $url within ${wait_s}s"; }
+
+echo "app-smoke: test: $test_cmd"
+if ! BASE_URL="${url%/}" bash -c "$test_cmd"; then
+  show_server_log
+  die "the smoke test failed"
+fi
+echo "app-smoke: passed"

package/templates/.harness/gates.conf.example

@@ -29,3 +29,12 @@ GATE_EXTRA=(
   # "coverage:npm run test:coverage -- --coverage.thresholds.lines=80"
   # "security:npm audit --audit-level=high"
 )
+
+# Optional final gates (ADR 0024): slow end-to-end checks ("name:command"). The task
+# loop runs them once, after the last task and before the change's PR opens; a
+# failure gets repair sessions. Never in a default run, the Stop hook or per task.
+# `.harness/gates.sh --final` (or `gates.sh <name>`) runs them by hand or in CI.
+# .harness/app-smoke.sh builds, starts the app on a seeded database and runs a test.
+GATE_FINAL=(
+  # "smoke:.harness/app-smoke.sh --if-exists scripts/smoke-test.ts --migrate '...' --seed '...' --build '...' --start '...' --url http://127.0.0.1:3100 --test '...'"
+)

package/templates/.harness/gates.sh

@@ -2,10 +2,14 @@
 # Runs every Definition-of-Done gate declared in .harness/gates.conf.
 # Exit 0 only if all gates pass. Used by the Stop hook, the task loop and CI.
 #
-# Usage: .harness/gates.sh [--stop] [gate ...]
-#   no args   run every gate (task loop, CI)
+# Usage: .harness/gates.sh [--stop|--final] [gate ...]
+#   no args   run every gate but the final ones (task loop, CI)
 #   --stop    run only STOP_GATES from gates.conf (fast profile for the Stop hook)
-#   gate ...  run only the named gates
+#   --final   run only GATE_FINAL from gates.conf: slow end-to-end checks that run
+#             once, at the end of a change in the task loop, before its PR opens
+#             (ADR 0024); `gates.sh --final` runs them in CI or by hand
+#   --has-final  exit 0 when GATE_FINAL is not empty, 1 otherwise (runs nothing)
+#   gate ...  run only the named gates (final ones included)
 #
 # HARNESS_SKIP_GATES="name ..." skips the named gates on this machine (e.g.
 # integration where no container runtime exists). Skips are printed on every
@@ -25,12 +29,22 @@ REQUIRED_GATES=(test lint typecheck build)
 GATE_EXTRA=()
 GATE_SETUP=""
 STOP_GATES=()
+GATE_FINAL=()
 # shellcheck source=/dev/null
 source "$conf"
 
 selected=("$@")
 if [[ "${1-}" == "--stop" ]]; then
   selected=("${STOP_GATES[@]}")
+elif [[ "${1-}" == "--has-final" ]]; then
+  [[ ${#GATE_FINAL[@]} -gt 0 ]]; exit
+elif [[ "${1-}" == "--final" ]]; then
+  selected=()
+  for entry in "${GATE_FINAL[@]}"; do selected+=("${entry%%:*}"); done
+  if [[ ${#selected[@]} -eq 0 ]]; then
+    echo "gates: no final gates configured (GATE_FINAL)"
+    exit 0
+  fi
 fi
 
 log_dir="$root/.harness/state/gates"
@@ -49,9 +63,18 @@ for entry in "${GATE_EXTRA[@]}"; do
   names+=("${entry%%:*}")
   cmds+=("${entry#*:}")
 done
+# Final gates run only on request (--final or by name), never in a default run.
+final_names=" "
+for entry in "${GATE_FINAL[@]}"; do
+  names+=("${entry%%:*}")
+  cmds+=("${entry#*:}")
+  final_names+="${entry%%:*} "
+done
 
 is_selected() {
-  [[ ${#selected[@]} -eq 0 ]] && return 0
+  if [[ ${#selected[@]} -eq 0 ]]; then
+    [[ "$final_names" != *" $1 "* ]]; return
+  fi
   local s
   for s in "${selected[@]}"; do [[ "$s" == "$1" ]] && return 0; done
   return 1

package/templates/.harness/lib.sh

@@ -93,9 +93,11 @@ harness_claude_flags() {
 
 # harness_usage_error <session.json>: the session ended on an account usage
 # limit or on missing usage credits ("You're out of usage credits. Switch to
-# another model..."), not on the work.
+# another model..."), not on the work. Claude Code names the window in the
+# message ("You've hit your 5-hour limit", "...your weekly limit", "...your
+# session limit") and sets api_error to usage_limit_reached, so both are matched.
 harness_usage_error() {
-  grep -qiE "hit your (session|usage) limit|usage limit reached|out of usage credits|credit balance is too low" "$1" 2>/dev/null
+  grep -qiE "hit your [^\"]{0,40}limit|usage_limit_reached|usage limit reached|out of usage credits|credit balance is too low" "$1" 2>/dev/null
 }
 
 # harness_level_descend <level>: the level one below, for the fallback after a

package/templates/.harness/models.conf

@@ -6,20 +6,21 @@
 #
 # Overrides, applied in this order after this file:
 #   .harness/models.local.conf   per project (project-owned, never synced)
-#   $HARNESS_MODELS_FILE         per machine (e.g. no Fable: R4="--model opus --effort xhigh")
+#   $HARNESS_MODELS_FILE         per machine (e.g. a cheaper R3: R3="--model opus --effort low")
 #
 # Rollout (decision 7): with HARNESS_MODEL_ROUTING=measure (the default) the
 # levels are computed and recorded but no flags are passed, so sessions keep
 # the account's default model; set HARNESS_MODEL_ROUTING=on (env file) to switch.
 # shellcheck shell=bash disable=SC2034
 
-R4="--model fable --effort high"   # design: specs, decisions no gate checks
-R3="--model opus --effort high"    # hard: ambiguous work, unclear diagnosis
+# No level uses Fable (amendment 2026-10-31 to ADR 0020).
+R4="--model opus --effort high"    # reserved: large product documents, last rung of the ladder
+R3="--model opus --effort medium"  # design/hard: specs, ambiguous work, unclear diagnosis
 R2="--model sonnet --effort medium" # build: a well-specified, gate-verified task
 R1="--model haiku --effort medium"  # routine: a mechanical, local edit
 
-TASK_spec_author=R4      # /opsx:propose in an interactive or channel session
-TASK_spec_adjust=R4      # spec PR review comments (watcher, ADR 0017)
+TASK_spec_author=R3      # /opsx:propose in an interactive or channel session; R4 for big product documents
+TASK_spec_adjust=R2      # spec PR review comments (watcher, ADR 0017): bounded edits to an existing spec
 TASK_impl=R2             # task loop (run-tasks.sh); tasks.md may override
 TASK_conflict=R3         # merge conflicts on a PR (watcher, ADR 0019)
 TASK_ci_fix=R2           # failing checks, clear cause (watcher, ADR 0019)
@@ -27,5 +28,5 @@ TASK_ci_fix_retry=R3     # failing checks after a failed fix (watcher, ADR 0019)
 TASK_review_comments=R2  # owner comments on implementation/sync PRs (ADR 0019, 1b)
 TASK_diagnosis=R3        # explaining an escalation
 TASK_review=R3           # PR review
-TASK_archive_fix=R2      # editorial split of a merged requirement over 500 characters (ADR 0022)
+TASK_archive_fix=R1      # editorial split of a merged requirement over 500 characters (ADR 0022), gate-verified
 TASK_routine=R1          # hand edits around sync and archive

package/templates/.harness/run-tasks.sh

@@ -16,6 +16,10 @@
 # one level down (R4 → R3 → R2 → R1) instead of pausing; the summary records
 # that session as "fallback" and the next at the level it ran. Pausing (exit 3)
 # is kept at R1 and with routing off.
+# After the last task, the final gates (GATE_FINAL in gates.conf, ADR 0024:
+# slow end-to-end checks such as the app smoke test) run once, before the PR is
+# opened; a failure gets up to HARNESS_FINAL_ATTEMPTS repair sessions with the
+# output as feedback, then escalates. A change with no GATE_FINAL skips the step.
 # Before the first task it installs the dependencies the spec declares
 # (.harness/deps.sh, ADR 0015) in their own commit; an agent that changes
 # dependency files escalates the run.
@@ -47,13 +51,14 @@
 #                            dependencies from the lockfile); HARNESS_MAIN_CHECKOUT
 #                            points at the main checkout, e.g. to copy a .env
 #   HARNESS_MAX_ATTEMPTS     attempts per task (default 4: the level ladder, ADR 0020)
+#   HARNESS_FINAL_ATTEMPTS   repair sessions for failing final gates (default 2)
 #   HARNESS_TASK_TIMEOUT     max seconds per agent session (default 1800)
 #   HARNESS_PERMISSION_MODE  claude --permission-mode (default acceptEdits)
 #   HARNESS_CLAUDE_ARGS      extra args for claude (experiments; models come from models.conf)
 #   HARNESS_COMMIT_TYPE      conventional commit type (default feat)
 set -euo pipefail
 
-usage() { sed -n '2,50p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }
+usage() { sed -n '2,54p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }
 die() { echo "run-tasks: $*" >&2; exit 1; }
 
 [[ $# -ge 1 ]] || usage 1
@@ -436,6 +441,81 @@ while line="$(next_task)"; [[ -n "$line" ]]; do
   fi
 done
 
+# ---- final gates (ADR 0024): once, after the last task, before the PR opens.
+# They are slow (build, server, database), so they are not part of every task's
+# gates. A failure is repaired like a task: the agent gets the output, the loop
+# re-runs the gates. HARNESS_SKIP_GATES skips them like any other gate.
+final_attempts="${HARNESS_FINAL_ATTEMPTS:-2}"
+final_gate_out=""
+if [[ -x .harness/gates.sh ]] && .harness/gates.sh --has-final; then
+  echo "==> final gates"
+  final_ok=0
+  for (( attempt = 0; attempt <= final_attempts; attempt++ )); do
+    if (( attempt > 0 )); then
+      level="$(harness_task_level impl "")" || level="R2"
+      level="$(harness_level_climb "$level" "$attempt")"
+      level_flags="$(harness_level_flags "$level")"
+      cur_task_id="final"; cur_task="final gates"; cur_attempt="$attempt"; attempt_started="$(date +%s)"
+      write_state
+      echo "    repair $attempt/$final_attempts at $level${level_flags:+ ($level_flags)}"
+      start=$(date +%s)
+      session_json="$log_dir/task-final-attempt-$attempt.json"
+      # shellcheck disable=SC2086
+      build_prompt "make the final gates pass: .harness/gates.sh --final (end-to-end checks run once at the end of the change). Fix the code or tests the change touched; the approved spec is the reference." "$final_gate_out" \
+        | timeout "$timeout_s" "${HARNESS_CLAUDE[@]}" -p --output-format json --permission-mode "$permission_mode" \
+          --disallowedTools "Bash(git commit:*)" "Bash(git push:*)" "Bash(.harness/spec-pr.sh:*)" \
+            "Bash(pnpm add:*)" "Bash(pnpm remove:*)" "Bash(pnpm update:*)" "Bash(npm install:*)" \
+          ${level_flags} ${HARNESS_CLAUDE_ARGS:-} >"$session_json" 2>&1 || true
+      warn_if_untrusted "$session_json"
+      if harness_usage_error "$session_json"; then
+        record final "$attempt" "$(( $(date +%s) - start ))" paused
+        run_status="paused"
+        git checkout -- . && git clean -fdq
+        echo "run-tasks: the agent session hit the account usage limit; pausing." >&2
+        echo "  The final gates are not done; re-run after the limit resets." >&2
+        exit 3
+      fi
+      git checkout -- "$change_dir"
+      git clean -fdq -- "$change_dir"
+      if [[ -n "$(git status --porcelain -- "${dep_files[@]}")" ]]; then
+        record final "$attempt" "$(( $(date +%s) - start ))" deps
+        run_status="escalated"
+        echo "run-tasks: the final-gates repair changed dependency files; undeclared dependencies need a spec change." >&2
+        exit 2
+      fi
+    fi
+    # A repair must keep every other gate green too.
+    if (( attempt > 0 )) && ! gate_out="$(.harness/gates.sh 2>&1)"; then
+      record final "$attempt" "$(( $(date +%s) - start ))" fail
+      final_gate_out="$(tail -n 120 <<<"$gate_out")"
+      echo "    FAIL (the repair broke the regular gates)"
+      continue
+    fi
+    if final_gate_out="$(.harness/gates.sh --final 2>&1)"; then
+      (( attempt == 0 )) || record final "$attempt" "$(( $(date +%s) - start ))" pass
+      final_ok=1
+      break
+    fi
+    printf '%s\n' "$final_gate_out" >"$log_dir/final-gates-attempt-$attempt.log"
+    (( attempt == 0 )) || record final "$attempt" "$(( $(date +%s) - start ))" fail
+    final_gate_out="$(tail -n 120 <<<"$final_gate_out")"
+    echo "    FAIL (final gates)"
+  done
+  if [[ $final_ok -eq 0 ]]; then
+    run_status="escalated"
+    echo "run-tasks: the final gates still fail after $final_attempts repair attempts; escalating." >&2
+    echo "  Work in progress is left on branch $branch in $wt. Logs: $log_dir/final-gates-attempt-*.log" >&2
+    exit 2
+  fi
+  if [[ -n "$(git status --porcelain)" ]]; then
+    git add -A
+    printf 'fix(%s): make the final gates pass\n\nRepaired by the task loop after .harness/gates.sh --final failed:\n\n%s\n' \
+      "$change" "$(head -n 20 "$log_dir/final-gates-attempt-0.log" 2>/dev/null)" | fold -s -w 72 | git commit -q -F -
+    echo "    final gates repaired → committed"
+  fi
+  echo "    final gates PASS"
+fi
+
 run_status="done"
 cur_task_id=""; cur_task=""; cur_attempt=0; attempt_started=""
 echo "run-tasks: all tasks in $tasks are done."

package/templates/stacks/nextjs/gates.conf

@@ -24,6 +24,10 @@ GATE_BUILD="$bin/next build"
 # Formats openspec/ after an archive (the archive output is not formatted).
 FORMAT_SPECS="$bin/prettier --write openspec"
 
+# Placeholder environment for the smoke test's build and server: no real login or
+# external service is involved (the test signs its own sessions with the secret).
+smoke_env="NEXTAUTH_URL=http://127.0.0.1:3100 NEXTAUTH_SECRET=harness-smoke-secret-not-used-anywhere-else GOOGLE_CLIENT_ID=smoke.apps.googleusercontent.com GOOGLE_CLIENT_SECRET=smoke NEXT_TELEMETRY_DISABLED=1"
+
 # Stop hook runs the fast gates (~40s); build (~60s) runs in the loop and CI.
 STOP_GATES=(lint typecheck test)
 
@@ -34,3 +38,13 @@ GATE_EXTRA=(
   "db:.harness/db-smoke.sh '$bin/prisma migrate deploy' '$bin/prisma db seed'"
   # "security:pnpm audit --audit-level high"
 )
+
+# Final gates (ADR 0024): slow end-to-end checks. The task loop runs them once, at
+# the end of a change, before its PR opens (not per task, not in the Stop hook);
+# `.harness/gates.sh --final` runs them by hand. The end-to-end smoke test needs a
+# migrated and seeded database and a running app; the gate passes by itself in a
+# project without scripts/smoke-test.ts. HARNESS_SKIP_GATES=smoke on a machine
+# without a container runtime; CI runs it in its own e2e job.
+GATE_FINAL=(
+  "smoke:.harness/app-smoke.sh --if-exists scripts/smoke-test.ts --migrate '$bin/prisma migrate deploy' --seed '$bin/prisma db seed' --build 'env $smoke_env $bin/next build' --start 'env $smoke_env $bin/next start -p 3100' --url http://127.0.0.1:3100 --test 'env $smoke_env $bin/tsx scripts/smoke-test.ts'"
+)