@hassangameryt/freeflow 0.3.0 → 0.5.0

package/evals/scripts/run-fixture-eval.sh

@@ -1,464 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-usage() {
-  cat >&2 <<'EOF'
-Usage:
-  run-fixture-eval.sh <eval-id> [--baseline|--with-skill] [--agent codex|claude]
-                      [--run-dir DIR] [--output FILE] [--dry-run]
-                      [--require-empty-diff] [--skill FILE ...]
-
-Direct fixture mode, for compatibility wrappers:
-  run-fixture-eval.sh --eval-id ID --fixture-dir DIR --prompt-file FILE
-                      --variant baseline|with-skill --run-dir DIR --output FILE
-                      [--agent codex|claude] [--skill FILE ...]
-EOF
-}
-
-fail() {
-  printf 'FAIL: %s\n' "$1" >&2
-  exit 1
-}
-
-absolute_existing_dir_file() {
-  local path="$1"
-  local dir
-  dir="$(cd "$(dirname "$path")" && pwd)"
-  printf '%s/%s\n' "$dir" "$(basename "$path")"
-}
-
-absolute_maybe_missing() {
-  local path="$1"
-  if [[ "$path" = /* ]]; then
-    printf '%s\n' "$path"
-  else
-    printf '%s/%s\n' "$repo_root" "$path"
-  fi
-}
-
-lower_id() {
-  printf '%s\n' "$1" | tr '[:upper:]' '[:lower:]'
-}
-
-shell_join() {
-  printf '%q' "$1"
-  shift || true
-  for arg in "$@"; do
-    printf ' %q' "$arg"
-  done
-}
-
-script_path="${BASH_SOURCE[0]}"
-script_dir="$(cd "$(dirname "$script_path")" && pwd)"
-repo_root="$(git -C "$script_dir" rev-parse --show-toplevel)"
-plugin_root="$repo_root"
-
-eval_id=""
-variant="with-skill"
-agent="${FREEFLOW_FIXTURE_AGENT:-codex}"
-registry="${FREEFLOW_FIXTURE_EVALS:-$plugin_root/evals/registries/fixture-evals.json}"
-requested_run_dir=""
-output_file=""
-fixture_dir=""
-prompt_file=""
-dry_run="${FREEFLOW_DRY_RUN:-0}"
-require_empty_diff="${FREEFLOW_REQUIRE_EMPTY_DIFF:-0}"
-skill_files=()
-
-while [ "$#" -gt 0 ]; do
-  case "$1" in
-    --eval-id)
-      eval_id="${2:-}"
-      shift 2
-      ;;
-    --variant)
-      variant="${2:-}"
-      shift 2
-      ;;
-    --baseline)
-      variant="baseline"
-      shift
-      ;;
-    --with-skill)
-      variant="with-skill"
-      shift
-      ;;
-    --agent)
-      agent="${2:-}"
-      shift 2
-      ;;
-    --registry)
-      registry="${2:-}"
-      shift 2
-      ;;
-    --run-dir)
-      requested_run_dir="${2:-}"
-      shift 2
-      ;;
-    --output)
-      output_file="${2:-}"
-      shift 2
-      ;;
-    --fixture-dir)
-      fixture_dir="${2:-}"
-      shift 2
-      ;;
-    --prompt-file)
-      prompt_file="${2:-}"
-      shift 2
-      ;;
-    --dry-run)
-      dry_run="1"
-      shift
-      ;;
-    --require-empty-diff)
-      require_empty_diff="1"
-      shift
-      ;;
-    --skill)
-      skill_files+=("${2:-}")
-      shift 2
-      ;;
-    --help|-h)
-      usage
-      exit 0
-      ;;
-    --)
-      shift
-      while [ "$#" -gt 0 ]; do
-        skill_files+=("$1")
-        shift
-      done
-      ;;
-    -*)
-      usage
-      fail "Unsupported option: $1"
-      ;;
-    *)
-      if [ -z "$eval_id" ]; then
-        eval_id="$1"
-      else
-        skill_files+=("$1")
-      fi
-      shift
-      ;;
-  esac
-done
-
-[ -n "$eval_id" ] || { usage; exit 2; }
-
-case "$variant" in
-  baseline|with-skill) ;;
-  *) fail "Unsupported variant: $variant. Expected baseline or with-skill." ;;
-esac
-
-case "$agent" in
-  codex|claude) ;;
-  *) fail "Unsupported agent: $agent. Expected codex or claude." ;;
-esac
-
-tmp_prompt_file=""
-trap 'if [ -n "$tmp_prompt_file" ]; then rm -f "$tmp_prompt_file"; fi' EXIT
-
-if [ -z "$fixture_dir" ] || [ -z "$prompt_file" ]; then
-  registry="$(absolute_maybe_missing "$registry")"
-  jq empty "$registry"
-
-  match_count="$(jq -r --arg id "$eval_id" '[.evals[] | select(.id == $id)] | length' "$registry")"
-  if [ "$match_count" != "1" ]; then
-    fail "Eval $eval_id must resolve to exactly one registry entry; got $match_count."
-  fi
-
-  eval_json="$(jq -cer --arg id "$eval_id" '.evals[] | select(.id == $id)' "$registry")"
-
-  fixture_root="$(jq -r '.fixture_root // empty' <<<"$eval_json")"
-  if [ -z "$fixture_root" ]; then
-    fixture_root="$(jq -r '.fixture_root // empty' "$registry")"
-  fi
-  [ -n "$fixture_root" ] || fail "Eval $eval_id has no fixture_root and registry has no default fixture_root."
-
-  if [ "$variant" = "baseline" ]; then
-    baseline_fixture_root="$(jq -r '.baseline_fixture_root // empty' <<<"$eval_json")"
-    if [ -n "$baseline_fixture_root" ]; then
-      fixture_root="$baseline_fixture_root"
-    fi
-  fi
-
-  fixture_dir="$plugin_root/$fixture_root"
-
-  prompt_ref="$(jq -r '.prompt_file // empty' <<<"$eval_json")"
-  if [ -n "$prompt_ref" ]; then
-    prompt_file="$plugin_root/$prompt_ref"
-  else
-    prompt="$(jq -r '.prompt // empty' <<<"$eval_json")"
-    [ -n "$prompt" ] || fail "Eval $eval_id has no prompt_file or prompt."
-
-    if [ "$dry_run" = "1" ]; then
-      prompt_file="<inline prompt from registry>"
-    else
-      tmp_prompt_file="$(mktemp "${TMPDIR:-/tmp}/freeflow-prompt-${eval_id}.XXXXXX")"
-      printf '%s\n' "$prompt" > "$tmp_prompt_file"
-      prompt_file="$tmp_prompt_file"
-    fi
-  fi
-else
-  fixture_dir="$(absolute_maybe_missing "$fixture_dir")"
-  prompt_file="$(absolute_maybe_missing "$prompt_file")"
-fi
-
-[ -d "$fixture_dir" ] || fail "Fixture directory does not exist: $fixture_dir"
-if [ "$prompt_file" != "<inline prompt from registry>" ]; then
-  [ -f "$prompt_file" ] || fail "Prompt file does not exist: $prompt_file"
-  prompt_file="$(absolute_existing_dir_file "$prompt_file")"
-fi
-fixture_dir="$(cd "$fixture_dir" && pwd)"
-
-eval_stem="$(lower_id "$eval_id")"
-if [ -z "$requested_run_dir" ]; then
-  requested_run_dir="evals/runs/manual/${eval_stem}-${variant}-${agent}"
-fi
-
-if [[ "$requested_run_dir" = /* ]]; then
-  run_dir="$requested_run_dir"
-else
-  run_dir="${TMPDIR:-/tmp}/freeflow-evals/${requested_run_dir}"
-fi
-
-if [ -z "$output_file" ]; then
-  output_file="evals/runs/manual/${eval_stem}-${variant}-${agent}-output.md"
-fi
-
-if [ "$dry_run" = "1" ]; then
-  resolved_output="$(absolute_maybe_missing "$output_file")"
-else
-  output_dir="$(dirname "$output_file")"
-  mkdir -p "$output_dir"
-  resolved_output="$(absolute_existing_dir_file "$output_file")"
-fi
-
-effective_skill_paths=()
-if [ "$variant" = "with-skill" ]; then
-  if [ "${#skill_files[@]}" -gt 0 ]; then
-    for skill_file in "${skill_files[@]}"; do
-      effective_skill_paths+=("$(absolute_maybe_missing "$skill_file")")
-    done
-  fi
-fi
-
-build_full_prompt() {
-  local prompt
-  prompt="$(cat "$prompt_file")"
-
-  local skill_block
-  if [ "$variant" = "with-skill" ]; then
-    skill_block="Use these skill files as active instructions before acting:"
-    if [ "${#effective_skill_paths[@]}" -gt 0 ]; then
-      for skill_path in "${effective_skill_paths[@]}"; do
-        skill_block="${skill_block}
-- ${skill_path}"
-      done
-    fi
-  else
-    skill_block="Do not read or use freeflow skill files."
-  fi
-
-  full_prompt="${skill_block}
-
-Work only inside this fixture directory:
-${run_dir}
-
-User prompt:
-${prompt}
-
-Requirements:
-- Work like a normal coding agent.
-- Inspect relevant files as needed.
-- Do not edit outside the fixture directory.
-- Return the final response you would send to the user.
-- Do not mention this is an eval."
-}
-
-prepare_run_dir() {
-  [ -n "$run_dir" ] || fail "Run directory is empty."
-  [ "$run_dir" != "/" ] || fail "Refusing to use / as run directory."
-
-  rm -rf "$run_dir"
-  mkdir -p "$(dirname "$run_dir")"
-  cp -R "$fixture_dir" "$run_dir"
-
-  if [ -f "$run_dir/.freeflow-eval-setup.sh" ]; then
-    (cd "$run_dir" && bash .freeflow-eval-setup.sh)
-  fi
-}
-
-capture_evidence() {
-  local output_stem="${resolved_output%.md}"
-  local diff_file="${output_stem}.diff"
-
-  diff -ru -x .git -x git-meta -x .freeflow-eval-setup.sh "$fixture_dir" "$run_dir" > "$diff_file" || true
-
-  if git -C "$run_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
-    git -C "$run_dir" status --short > "${output_stem}.git-status.txt" || true
-    git -C "$run_dir" log --oneline --max-count=5 > "${output_stem}.git-log.txt" || true
-    git -C "$run_dir" show --stat --oneline --name-only HEAD > "${output_stem}.git-head.txt" || true
-  fi
-
-  if [ "$require_empty_diff" = "1" ] && [ -s "$diff_file" ]; then
-    echo "Diff is not empty; failing because FREEFLOW_REQUIRE_EMPTY_DIFF=1 or --require-empty-diff was set." >&2
-    return 1
-  fi
-}
-
-write_metadata() {
-  local output_stem="${resolved_output%.md}"
-  local skills_json
-  if [ "${#effective_skill_paths[@]}" -eq 0 ]; then
-    skills_json="[]"
-  else
-    skills_json="$(printf '%s\n' "${effective_skill_paths[@]}" | jq -R -s 'split("\n") | map(select(length > 0))')"
-  fi
-
-  jq -n \
-    --arg eval_id "$eval_id" \
-    --arg variant "$variant" \
-    --arg agent "$agent" \
-    --arg registry "$registry" \
-    --arg fixture_dir "$fixture_dir" \
-    --arg run_dir "$run_dir" \
-    --arg output "$resolved_output" \
-    --arg diff "${output_stem}.diff" \
-    --arg exit_status "${output_stem}.exit-status.txt" \
-    --argjson skills "$skills_json" \
-    '{
-      eval_id: $eval_id,
-      variant: $variant,
-      agent: $agent,
-      registry: $registry,
-      fixture_dir: $fixture_dir,
-      run_dir: $run_dir,
-      output: $output,
-      diff: $diff,
-      exit_status: $exit_status,
-      skills: $skills
-    }' > "${output_stem}.metadata.json"
-}
-
-print_dry_run() {
-  printf 'eval_id=%s\n' "$eval_id"
-  printf 'agent=%s\n' "$agent"
-  printf 'runner=%s\n' "$script_path"
-  printf 'adapter=%s\n' "$agent"
-  printf 'variant=%s\n' "$variant"
-  printf 'fixture_action=would copy\n'
-  printf 'fixture=%s\n' "$fixture_dir"
-  printf 'prompt=%s\n' "$prompt_file"
-  printf 'run_dir=%s\n' "$run_dir"
-  printf 'output=%s\n' "$resolved_output"
-  printf 'require_empty_diff=%s\n' "$require_empty_diff"
-  if [ "$variant" = "baseline" ]; then
-    printf 'runtime_context_disabled=1\n'
-  fi
-  printf 'skills=%s\n' "${effective_skill_paths[*]-}"
-}
-
-run_codex_adapter() {
-  local codex_sandbox="${FREEFLOW_CODEX_SANDBOX:-workspace-write}"
-  local command=(codex exec --cd "$run_dir" --sandbox "$codex_sandbox" --skip-git-repo-check --output-last-message "$resolved_output")
-
-  if [ "$dry_run" = "1" ]; then
-    print_dry_run
-    printf 'command='
-    shell_join "${command[@]}"
-    printf ' <prompt>\n'
-    return 0
-  fi
-
-  build_full_prompt
-
-  set +e
-  if [ "$variant" = "baseline" ]; then
-    FREEFLOW_DISABLE_RUNTIME_CONTEXT=1 "${command[@]}" "$full_prompt"
-  else
-    "${command[@]}" "$full_prompt"
-  fi
-  agent_status="$?"
-  set -e
-  printf '%s\n' "$agent_status" > "${resolved_output%.md}.exit-status.txt"
-  return "$agent_status"
-}
-
-run_claude_adapter() {
-  local default_plugin_dir="none"
-  if [ "$variant" = "with-skill" ]; then
-    default_plugin_dir="$plugin_root"
-  fi
-
-  local claude_plugin_dir="${FREEFLOW_CLAUDE_PLUGIN_DIR:-$default_plugin_dir}"
-  local claude_args=(
-    -p
-    --no-session-persistence
-    --permission-mode "${FREEFLOW_CLAUDE_PERMISSION_MODE:-dontAsk}"
-    --tools "${FREEFLOW_CLAUDE_TOOLS:-Read,Edit,Bash}"
-    --add-dir "$run_dir"
-  )
-
-  if [ "${FREEFLOW_CLAUDE_BARE:-1}" = "1" ]; then
-    claude_args+=(--bare)
-  fi
-
-  if [ "$claude_plugin_dir" != "none" ]; then
-    claude_args+=(--plugin-dir "$claude_plugin_dir")
-  fi
-
-  if [ "$dry_run" = "1" ]; then
-    print_dry_run
-    printf 'bare=%s\n' "${FREEFLOW_CLAUDE_BARE:-1}"
-    printf 'plugin_dir=%s\n' "$claude_plugin_dir"
-    printf 'command=claude'
-    printf ' %q' "${claude_args[@]}"
-    printf ' <prompt>\n'
-    return 0
-  fi
-
-  build_full_prompt
-
-  local stderr_file="${resolved_output%.md}.stderr.txt"
-
-  set +e
-  if [ "$variant" = "baseline" ]; then
-    FREEFLOW_DISABLE_RUNTIME_CONTEXT=1 claude "${claude_args[@]}" "$full_prompt" > "$resolved_output" 2> "$stderr_file"
-  else
-    claude "${claude_args[@]}" "$full_prompt" > "$resolved_output" 2> "$stderr_file"
-  fi
-  agent_status="$?"
-  set -e
-  printf '%s\n' "$agent_status" > "${resolved_output%.md}.exit-status.txt"
-  return "$agent_status"
-}
-
-if [ "$dry_run" = "1" ]; then
-  case "$agent" in
-    codex) run_codex_adapter ;;
-    claude) run_claude_adapter ;;
-  esac
-  exit 0
-fi
-
-write_metadata
-prepare_run_dir
-
-agent_status=0
-case "$agent" in
-  codex) run_codex_adapter || agent_status="$?" ;;
-  claude) run_claude_adapter || agent_status="$?" ;;
-esac
-
-evidence_status=0
-capture_evidence || evidence_status="$?"
-
-if [ "$agent_status" -ne 0 ]; then
-  echo "$agent exited with status $agent_status. See $resolved_output and sidecar files." >&2
-  exit "$agent_status"
-fi
-
-exit "$evidence_status"