@kontourai/flow-agents 0.1.1

package/evals/lib/assertions/verify-after-fix.js

@@ -0,0 +1,61 @@
+// verify-after-fix.js — Assert that any code change during review/verify is followed by a clean verification pass
+const { getNewEvents, filterByType, getToolInvocations, getSubagentCalls } = require('./telemetry-utils');
+
+const WRITE_TOOLS = new Set(['write files', 'write', 'apply_patch', 'edit']);
+const REVIEW_AGENTS = new Set(['tool-code-reviewer', 'tool-security-reviewer']);
+const VERIFY_AGENTS = new Set(['tool-verifier', 'tool-playwright']);
+const REPORTER_AGENTS = new Set([...REVIEW_AGENTS, ...VERIFY_AGENTS]);
+
+module.exports = (output, { config }) => {
+  const events = getNewEvents();
+  const toolEvents = getToolInvocations(events);
+  const subagentCalls = getSubagentCalls(events);
+  const violations = [];
+
+  // Check 1: Reviewers/verifiers must never invoke write tools
+  const reporterWrites = toolEvents.filter(e => {
+    const agent = e.agent && e.agent.name;
+    const tool = e.tool && e.tool.name && String(e.tool.name).toLowerCase();
+    return agent && REPORTER_AGENTS.has(agent) && WRITE_TOOLS.has(tool);
+  });
+
+  if (reporterWrites.length > 0) {
+    violations.push(
+      `Reporter agents wrote code: ${reporterWrites.map(e => `${e.agent.name} → ${e.tool.name}`).join('; ')}`
+    );
+  }
+
+  // Check 2: After any write tool call, there must be a subsequent tool-verifier delegation
+  const allEvents = events;
+  let lastWriteIdx = -1;
+  let lastVerifyIdx = -1;
+
+  for (let i = 0; i < allEvents.length; i++) {
+    const e = allEvents[i];
+    const toolName = e.tool && e.tool.name && String(e.tool.name).toLowerCase();
+    if (e.event_type === 'tool.invoke' && e.tool && WRITE_TOOLS.has(toolName)) {
+      lastWriteIdx = i;
+    }
+    if (e.event_type === 'tool.invoke' && e.tool && toolName === 'delegate to a specialist agent' &&
+        e.tool.input && e.tool.input.command === 'InvokeSubagents') {
+      const subs = e.tool.input.content && e.tool.input.content.subagents;
+      if (subs && subs.some(s => s.agent_name === 'tool-verifier')) {
+        lastVerifyIdx = i;
+      }
+    }
+    if (e.event_type === 'tool.invoke' && e.tool && toolName === 'spawn_agent' &&
+        e.tool.input && e.tool.input.agent_type === 'tool-verifier') {
+      lastVerifyIdx = i;
+    }
+  }
+
+  if (lastWriteIdx > lastVerifyIdx) {
+    violations.push('Code was written after the last verification pass — missing re-verify');
+  }
+
+  if (violations.length === 0) {
+    return { pass: true, score: 1, reason: 'No code changes without subsequent verification' };
+  }
+
+  return { pass: false, score: 0, reason: `Re-verify violations: ${violations.join('; ')}` };
+};

package/evals/lib/claude-judge.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# claude-judge.sh — Promptfoo exec provider for llm-rubric judging via Claude Code.
+set -euo pipefail
+
+PROMPT="${1:-}"
+TIMEOUT="${CLAUDE_EVAL_JUDGE_TIMEOUT:-180}"
+MAX_LEN=200000
+if [[ ${#PROMPT} -gt $MAX_LEN ]]; then
+  PROMPT="${PROMPT:0:$MAX_LEN}... [truncated for eval - output exceeded ${MAX_LEN} chars]"
+fi
+
+OUT="$(mktemp /tmp/flow-agents-claude-judge.XXXXXX)"
+LOG="$(mktemp /tmp/flow-agents-claude-judge-log.XXXXXX)"
+trap 'rm -f "$OUT" "$LOG"' EXIT
+
+if ! command -v claude >/dev/null 2>&1; then
+  echo "claude CLI is not installed or not on PATH" >&2
+  exit 2
+fi
+
+if command -v timeout >/dev/null 2>&1; then
+  TIMEOUT_CMD=(timeout "$TIMEOUT")
+elif command -v gtimeout >/dev/null 2>&1; then
+  TIMEOUT_CMD=(gtimeout "$TIMEOUT")
+else
+  TIMEOUT_CMD=()
+fi
+
+"${TIMEOUT_CMD[@]}" claude \
+  -p \
+  --permission-mode bypassPermissions \
+  --add-dir /tmp \
+  --output-format text \
+  "$PROMPT" >"$OUT" 2>"$LOG" || {
+    cat "$OUT" 2>/dev/null
+    sed -n '1,120p' "$LOG" >&2
+    exit 1
+  }
+
+cat "$OUT"

package/evals/lib/claude-provider.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# claude-provider.sh — Promptfoo exec provider that runs Flow Agents through Claude Code.
+set -euo pipefail
+
+PROMPT="${1:-}"
+OPTIONS="${2:-}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+TIMEOUT="${CLAUDE_EVAL_TIMEOUT:-300}"
+FLUSH_SLEEP="${FLOW_AGENTS_EVAL_TELEMETRY_FLUSH_SLEEP:-0.5}"
+SNAPSHOT_FILE="${FLOW_AGENTS_EVAL_TELEMETRY_SNAPSHOT:-/tmp/promptfoo-eval-telemetry-snapshot.txt}"
+TELEMETRY_FILE_MARKER="${FLOW_AGENTS_EVAL_TELEMETRY_FILE_MARKER:-/tmp/promptfoo-eval-telemetry-file.txt}"
+
+AGENT=""
+if [[ -n "$OPTIONS" ]]; then
+  AGENT=$(node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const j=JSON.parse(d);process.stdout.write(j.config?.agent||'')}catch{}})" <<<"$OPTIONS" 2>/dev/null || true)
+fi
+AGENT="${AGENT:-${FLOW_AGENTS_EVAL_AGENT:-dev}}"
+
+if ! command -v claude >/dev/null 2>&1; then
+  echo "claude CLI is not installed or not on PATH" >&2
+  exit 2
+fi
+
+run_claude() {
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$TIMEOUT" "${CLAUDE_CMD[@]}"
+  elif command -v gtimeout >/dev/null 2>&1; then
+    gtimeout "$TIMEOUT" "${CLAUDE_CMD[@]}"
+  else
+    "${CLAUDE_CMD[@]}"
+  fi
+}
+
+prepare_workdir() {
+  local work_root="${CLAUDE_EVAL_WORK_ROOT:-/tmp/flow-agents-claude-eval}"
+  local work_dir="$work_root/$AGENT"
+  rm -rf "$work_dir"
+  mkdir -p "$work_dir"
+  (cd "$ROOT_DIR" && flow_agents_node scripts/build-universal-bundles.js >/dev/null)
+  bash "$ROOT_DIR/dist/claude-code/install.sh" "$work_dir" >/dev/null
+  mkdir -p "$work_dir/.telemetry"
+  echo "$work_dir"
+}
+
+WORK_DIR="$(prepare_workdir)"
+TELEMETRY_FILE="$WORK_DIR/.telemetry/full.jsonl"
+echo "$TELEMETRY_FILE" > "$TELEMETRY_FILE_MARKER"
+if [[ -f "$TELEMETRY_FILE" ]]; then
+  wc -l < "$TELEMETRY_FILE" | tr -d ' ' > "$SNAPSHOT_FILE"
+else
+  echo "0" > "$SNAPSHOT_FILE"
+fi
+
+CLAUDE_CMD=(
+  env
+  FLOW_AGENTS_CLAUDE_TELEMETRY_CHANNELS="${FLOW_AGENTS_CLAUDE_TELEMETRY_CHANNELS:-full,analytics}"
+  claude
+  -p
+  --agent "$AGENT"
+  --permission-mode bypassPermissions
+  --add-dir "$WORK_DIR"
+  --output-format text
+  "$PROMPT"
+)
+
+set +e
+RAW=$(cd "$WORK_DIR" && run_claude 2>&1)
+STATUS=$?
+set -e
+sleep "$FLUSH_SLEEP"
+echo "$RAW" | sed $'s/\x1b\[[0-9;]*[a-zA-Z]//g; s/\x1b\[[0-9;]*m//g; s/\x07//g' \
+  | grep -v '^\s*$'
+exit "$STATUS"

package/evals/lib/codex-judge.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# codex-judge.sh — Promptfoo exec provider for llm-rubric judging via Codex.
+set -euo pipefail
+
+PROMPT="${1:-}"
+TIMEOUT="${CODEX_EVAL_JUDGE_TIMEOUT:-180}"
+MAX_LEN=200000
+if [[ ${#PROMPT} -gt $MAX_LEN ]]; then
+  PROMPT="${PROMPT:0:$MAX_LEN}... [truncated for eval - output exceeded ${MAX_LEN} chars]"
+fi
+
+OUT="$(mktemp /tmp/flow-agents-codex-judge.XXXXXX)"
+LOG="$(mktemp /tmp/flow-agents-codex-judge-log.XXXXXX)"
+trap 'rm -f "$OUT" "$LOG"' EXIT
+
+if command -v timeout >/dev/null 2>&1; then
+  TIMEOUT_CMD=(timeout "$TIMEOUT")
+elif command -v gtimeout >/dev/null 2>&1; then
+  TIMEOUT_CMD=(gtimeout "$TIMEOUT")
+else
+  TIMEOUT_CMD=()
+fi
+
+"${TIMEOUT_CMD[@]}" codex exec \
+  --ignore-user-config \
+  --skip-git-repo-check \
+  -C /tmp \
+  --sandbox read-only \
+  --json \
+  -c model='"gpt-5.5"' \
+  -c model_reasoning_effort='"medium"' \
+  --output-last-message "$OUT" \
+  "$PROMPT" >"$LOG" 2>&1 || {
+    cat "$OUT" 2>/dev/null
+    sed -n '1,120p' "$LOG" >&2
+    exit 1
+  }
+
+cat "$OUT"

package/evals/lib/codex-provider.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# codex-provider.sh — Promptfoo exec provider that runs Flow Agents through Codex.
+set -euo pipefail
+
+PROMPT="${1:-}"
+OPTIONS="${2:-}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+TIMEOUT="${CODEX_EVAL_TIMEOUT:-300}"
+FLUSH_SLEEP="${FLOW_AGENTS_EVAL_TELEMETRY_FLUSH_SLEEP:-0.5}"
+SNAPSHOT_FILE="${FLOW_AGENTS_EVAL_TELEMETRY_SNAPSHOT:-/tmp/promptfoo-eval-telemetry-snapshot.txt}"
+TELEMETRY_FILE_MARKER="${FLOW_AGENTS_EVAL_TELEMETRY_FILE_MARKER:-/tmp/promptfoo-eval-telemetry-file.txt}"
+
+AGENT=""
+if [[ -n "$OPTIONS" ]]; then
+  AGENT=$(node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const j=JSON.parse(d);process.stdout.write(j.config?.agent||'')}catch{}})" <<<"$OPTIONS" 2>/dev/null || true)
+fi
+AGENT="${AGENT:-${FLOW_AGENTS_EVAL_AGENT:-dev}}"
+
+profile_for_agent() {
+  case "$1" in
+    dev) echo "kdev" ;;
+    *) echo "" ;;
+  esac
+}
+
+
+strip_json_events() {
+  node -e "const rl=require('readline').createInterface({input:process.stdin});rl.on('line',l=>{if(!l)return;try{const o=JSON.parse(l);if(o.type==='agent_message'&&typeof o.text==='string')console.log(o.text);else if(o.type==='item.completed'&&o.item?.type==='agent_message'&&typeof o.item.text==='string')console.log(o.item.text)}catch{console.log(l)}})"
+}
+
+run_codex() {
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$TIMEOUT" "${CODEX_CMD[@]}" "$PROMPT"
+  elif command -v gtimeout >/dev/null 2>&1; then
+    gtimeout "$TIMEOUT" "${CODEX_CMD[@]}" "$PROMPT"
+  else
+    "${CODEX_CMD[@]}" "$PROMPT"
+  fi
+}
+
+prepare_workdir() {
+  local work_root="${CODEX_EVAL_WORK_ROOT:-/tmp/flow-agents-codex-eval}"
+  local work_dir="$work_root/$AGENT"
+  rm -rf "$work_dir"
+  mkdir -p "$work_dir"
+  (cd "$ROOT_DIR" && flow_agents_node scripts/build-universal-bundles.js >/dev/null)
+  cp -R "$ROOT_DIR/dist/codex/." "$work_dir/"
+  cp "$work_dir/.codex/config.toml" "$work_dir/.codex/config-eval.toml"
+  for auth_file in auth.json version.json installation_id; do
+    if [[ -f "${CODEX_REAL_HOME:-$HOME/.codex}/$auth_file" ]]; then
+      cp "${CODEX_REAL_HOME:-$HOME/.codex}/$auth_file" "$work_dir/.codex/$auth_file"
+    fi
+  done
+  mkdir -p "$work_dir/.telemetry"
+  echo "$work_dir"
+}
+
+WORK_DIR="$(prepare_workdir)"
+TELEMETRY_FILE="$WORK_DIR/.telemetry/full.jsonl"
+echo "$TELEMETRY_FILE" > "$TELEMETRY_FILE_MARKER"
+if [[ -f "$TELEMETRY_FILE" ]]; then
+  wc -l < "$TELEMETRY_FILE" | tr -d ' ' > "$SNAPSHOT_FILE"
+else
+  echo "0" > "$SNAPSHOT_FILE"
+fi
+
+PROFILE="$(profile_for_agent "$AGENT")"
+if [[ -n "$PROFILE" ]]; then
+  CODEX_CMD=(env CODEX_HOME="$WORK_DIR/.codex" codex -p "$PROFILE" exec --skip-git-repo-check -C "$WORK_DIR" --sandbox read-only --json)
+else
+  CODEX_CMD=(env CODEX_HOME="$WORK_DIR/.codex" codex -c "developer_instructions=$(node -e "const fs=require('fs'),p='$WORK_DIR/.codex/agents/$AGENT.toml';if(!fs.existsSync(p)){process.stdout.write('\"\"');process.exit(0)}const m=fs.readFileSync(p,'utf8').match(/^developer_instructions\\s*=\\s*(.+)$/m);process.stdout.write(m?m[1]:'\"\"')")" exec --skip-git-repo-check -C "$WORK_DIR" --sandbox read-only --json)
+fi
+
+set +e
+RAW=$(run_codex 2>&1)
+STATUS=$?
+set -e
+sleep "$FLUSH_SLEEP"
+echo "$RAW" | strip_json_events | sed $'s/\x1b\[[0-9;]*[a-zA-Z]//g; s/\x1b\[[0-9;]*m//g; s/\x07//g'
+exit "$STATUS"

package/evals/lib/eval-dev.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export FLOW_AGENTS_EVAL_AGENT=dev
+export KIRO_EVAL_AGENT=dev
+exec bash "$SCRIPT_DIR/eval-provider.sh" "$@"

package/evals/lib/eval-judge.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# eval-judge.sh — Runtime-neutral promptfoo rubric judge provider.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RUNTIME="${FLOW_AGENTS_EVAL_JUDGE_RUNTIME:-${FLOW_AGENTS_EVAL_RUNTIME:-${EVAL_RUNTIME:-kiro}}}"
+
+case "$RUNTIME" in
+  kiro|kiro-cli)
+    exec bash "$SCRIPT_DIR/kiro-judge.sh" "$@"
+    ;;
+  codex)
+    exec bash "$SCRIPT_DIR/codex-judge.sh" "$@"
+    ;;
+  claude|claude-code)
+    exec bash "$SCRIPT_DIR/claude-judge.sh" "$@"
+    ;;
+  *)
+    echo "Unsupported FLOW_AGENTS_EVAL_JUDGE_RUNTIME='$RUNTIME' (expected kiro, codex, or claude)" >&2
+    exit 2
+    ;;
+esac

package/evals/lib/eval-provider.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# eval-provider.sh — Runtime-neutral promptfoo subject provider.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RUNTIME="${FLOW_AGENTS_EVAL_RUNTIME:-${EVAL_RUNTIME:-kiro}}"
+AGENT="${FLOW_AGENTS_EVAL_AGENT:-${KIRO_EVAL_AGENT:-dev}}"
+
+export FLOW_AGENTS_EVAL_AGENT="$AGENT"
+export KIRO_EVAL_AGENT="$AGENT"
+
+case "$RUNTIME" in
+  kiro|kiro-cli)
+    exec bash "$SCRIPT_DIR/kiro-provider.sh" "$@"
+    ;;
+  codex)
+    exec bash "$SCRIPT_DIR/codex-provider.sh" "$@"
+    ;;
+  claude|claude-code)
+    exec bash "$SCRIPT_DIR/claude-provider.sh" "$@"
+    ;;
+  *)
+    echo "Unsupported FLOW_AGENTS_EVAL_RUNTIME='$RUNTIME' (expected kiro, codex, or claude)" >&2
+    exit 2
+    ;;
+esac

package/evals/lib/eval-report.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# eval-report.sh — Generate markdown eval report from promptfoo JSON output
+# Usage: bash lib/eval-report.sh <results-json> [previous-json]
+# Output: markdown report to stdout
+set -uo pipefail
+
+RESULTS="${1:?Usage: bash lib/eval-report.sh <results.json> [previous.json]}"
+PREVIOUS="${2:-}"
+
+if [[ ! -f "$RESULTS" ]]; then
+  echo "Error: Results file not found: $RESULTS" >&2
+  exit 1
+fi
+
+AGENT=$(basename "$RESULTS" | sed 's/-[0-9].*$//')
+DATE=$(date +%Y-%m-%d)
+
+# Extract stats via jq
+TOTAL=$(jq '.results.results | length' "$RESULTS")
+PASSED=$(jq '[.results.results[] | select(.success == true)] | length' "$RESULTS")
+FAILED=$((TOTAL - PASSED))
+PASS_RATE=$(echo "scale=0; $PASSED * 100 / $TOTAL" | bc 2>/dev/null || echo "N/A")
+
+# Check for repeat data
+REPEAT=$(jq -r '.results.stats.repeatCount // 0' "$RESULTS" 2>/dev/null || echo "0")
+
+cat <<EOF
+# Eval Report: ${AGENT} — ${DATE}
+
+## Summary
+- Cases: ${TOTAL} total
+- Passed: ${PASSED}/${TOTAL} (${PASS_RATE}%)
+- Failed: ${FAILED}
+EOF
+
+if [[ "$REPEAT" -gt 1 ]]; then
+  echo "- Repeat count: ${REPEAT} (pass@k computed per case)"
+fi
+
+echo ""
+echo "## Results"
+echo "| # | Prompt (truncated) | Pass | Assertions |"
+echo "|---|-------------------|------|------------|"
+
+jq -r '.results.results | to_entries[] | "\(.key + 1)|\(.value.vars.prompt // "N/A" | .[0:50])|\(.value.success)|\(.value.gradingResult.componentResults // [] | length) checked"' "$RESULTS" 2>/dev/null | \
+while IFS='|' read -r num prompt pass asserts; do
+  icon=$([[ "$pass" == "true" ]] && echo "✓" || echo "✗")
+  echo "| ${num} | ${prompt} | ${icon} | ${asserts} |"
+done
+
+# Failures section
+if [[ "$FAILED" -gt 0 ]]; then
+  echo ""
+  echo "## Failures"
+  jq -r '.results.results | to_entries[] | select(.value.success == false) | "### Case \(.key + 1): \(.value.vars.prompt // "N/A" | .[0:60])\n- Failing assertions: \([.value.gradingResult.componentResults[]? | select(.pass == false) | .assertion.type // "unknown" | select(.pass == false) | .assertion.type // "unknown"] | join(", "))\n"' "$RESULTS" 2>/dev/null
+fi
+
+# Trend comparison
+if [[ -n "$PREVIOUS" && -f "$PREVIOUS" ]]; then
+  PREV_PASSED=$(jq '[.results.results[] | select(.success == true)] | length' "$PREVIOUS")
+  PREV_TOTAL=$(jq '.results.results | length' "$PREVIOUS")
+  echo ""
+  echo "## Trend"
+  echo "- Previous: ${PREV_PASSED}/${PREV_TOTAL}"
+  echo "- Current:  ${PASSED}/${TOTAL}"
+  if [[ "$PASSED" -gt "$PREV_PASSED" ]]; then
+    echo "- Direction: ↑ improved"
+  elif [[ "$PASSED" -lt "$PREV_PASSED" ]]; then
+    echo "- Direction: ↓ regressed"
+  else
+    echo "- Direction: → stable"
+  fi
+fi

package/evals/lib/kiro-dev.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export KIRO_EVAL_AGENT=dev
+exec bash "$SCRIPT_DIR/kiro-provider.sh" "$@"

package/evals/lib/kiro-judge.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# kiro-judge.sh — Promptfoo exec provider for llm-rubric judging via kiro-cli
+set -o pipefail
+PROMPT="$1"
+
+# Truncate if too large for shell args (macOS limit ~262144 bytes)
+MAX_LEN=200000
+if [[ ${#PROMPT} -gt $MAX_LEN ]]; then
+  PROMPT="${PROMPT:0:$MAX_LEN}... [truncated for eval — output exceeded ${MAX_LEN} chars]"
+fi
+
+RAW=$(kiro-cli chat --no-interactive --trust-tools "" "$PROMPT" 2>/dev/null)
+echo "$RAW" | sed $'s/\x1b\[[0-9;]*[a-zA-Z]//g; s/\x1b\[[0-9;]*m//g; s/\x07//g' \
+  | grep -v '^\s*$' | grep -v 'hooks finished' | grep -v 'Credits:' \
+  | grep -v 'WARNING:' | grep -v 'All tools are now trusted' \
+  | grep -v 'Checkpoints are not' | grep -v 'Learn more at' \
+  | sed 's/^> //' | sed 's/^[[:space:]]*//'

package/evals/lib/kiro-provider.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# kiro-provider.sh — Promptfoo exec provider that runs kiro-cli agents
+# Usage: bash kiro-provider.sh <prompt> <options_json> <context_json>
+# Agent is determined from the prompt's {{agent}} variable passed via options JSON
+set -o pipefail
+
+PROMPT="$1"
+OPTIONS="$2"
+SNAPSHOT_FILE="/tmp/promptfoo-eval-telemetry-snapshot.txt"
+TIMEOUT="${KIRO_EVAL_TIMEOUT:-300}"
+
+# Extract agent from options JSON or env var
+if [[ -n "$OPTIONS" ]]; then
+  AGENT=$(node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const j=JSON.parse(d);process.stdout.write(j.config?.agent||'')}catch{}})" <<<"$OPTIONS" 2>/dev/null)
+fi
+AGENT="${AGENT:-${KIRO_EVAL_AGENT:-dev}}"
+
+# Auto-detect telemetry file from installed agent location
+_find_telemetry() {
+  local agent="$1"
+  for f in "$HOME/.kiro/agents/"*"-${agent}.json"; do
+    [[ -f "$f" ]] || continue
+    local pkg_path
+    pkg_path=$(grep -o "$HOME/.flow-agents\"]*" "$f" 2>/dev/null | head -1 | sed 's|/context/.*||')
+    if [[ -n "$pkg_path" && -f "$pkg_path/.telemetry/full.jsonl" ]]; then
+      echo "$pkg_path/.telemetry/full.jsonl"
+      return
+    fi
+  done
+  echo "$HOME/.flow-agents"
+}
+TELEMETRY_FILE="$(_find_telemetry "$AGENT")"
+
+SAFE_TOOLS="read files,code,grep,glob,knowledge,web_search,web_fetch,delegate to a specialist agent,todo tool,thinking,session,report_issue"
+
+# Snapshot telemetry line count before run
+if [[ -f "$TELEMETRY_FILE" ]]; then
+  wc -l < "$TELEMETRY_FILE" | tr -d ' ' > "$SNAPSHOT_FILE"
+else
+  echo "0" > "$SNAPSHOT_FILE"
+fi
+
+# Run agent, capture output
+RAW=$(timeout "$TIMEOUT" kiro-cli chat \
+  --agent "$AGENT" \
+  --no-interactive \
+  --trust-tools "$SAFE_TOOLS" \
+  "$PROMPT" 2>/dev/null)
+
+# Strip ANSI escape codes and bell chars
+CLEAN=$(echo "$RAW" | sed $'s/\x1b\[[0-9;]*[a-zA-Z]//g; s/\x1b\[[0-9;]*m//g; s/\x07//g')
+
+# Remove kiro chrome lines but keep the actual response content
+echo "$CLEAN" | grep -v '^\s*$' \
+  | grep -v 'hooks finished' \
+  | grep -v 'Credits:' \
+  | grep -v 'WARNING:' \
+  | grep -v 'All tools are now trusted' \
+  | grep -v 'Checkpoints are not' \
+  | grep -v 'Learn more at' \
+  | sed 's/^> //' \
+  | sed 's/^[[:space:]]*//'

package/evals/lib/node.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Shared command adapter for evals. Historical script entry paths are routed to TypeScript tools.
+
+FLOW_AGENTS_EVAL_ROOT="${ROOT:-${ROOT_DIR:-}}"
+
+flow_agents_build_ts() {
+  (cd "$FLOW_AGENTS_EVAL_ROOT" && npm run build --silent >/dev/null)
+}
+
+flow_agents_node() {
+  case "$1" in
+    */scripts/build-universal-bundles.js|scripts/build-universal-bundles.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" build-bundles "$@"
+      return
+      ;;
+    */scripts/generate-context-map.js|scripts/generate-context-map.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" context-map "$@"
+      return
+      ;;
+    */scripts/filter-installed-packs.js|scripts/filter-installed-packs.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" filter-installed-packs "$@"
+      return
+      ;;
+    workflow-sidecar)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/workflow-sidecar.js" "$@"
+      return
+      ;;
+    validate-workflow-artifacts)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/validate-workflow-artifacts.js" "$@"
+      return
+      ;;
+    */scripts/validate-source-tree.js|scripts/validate-source-tree.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/validate-source-tree.js" "$@"
+      return
+      ;;
+    */scripts/flow-kit.js|scripts/flow-kit.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/flow-kit.js" "$@"
+      return
+      ;;
+    */scripts/effective-backlog-settings.js|scripts/effective-backlog-settings.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/effective-backlog-settings.js" "$@"
+      return
+      ;;
+    */scripts/pull-work-provider.js|scripts/pull-work-provider.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/pull-work-provider.js" "$@"
+      return
+      ;;
+    workflow-artifact-cleanup-audit)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" workflow-artifact-cleanup-audit "$@"
+      return
+      ;;
+    fixture-retirement-audit)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" fixture-retirement-audit "$@"
+      return
+      ;;
+    */scripts/publish-change-helper.js|scripts/publish-change-helper.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/publish-change-helper.js" "$@"
+      return
+      ;;
+    */scripts/promote-workflow-artifact.js|scripts/promote-workflow-artifact.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/promote-workflow-artifact.js" "$@"
+      return
+      ;;
+    */scripts/usage-feedback.js|scripts/usage-feedback.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli/usage-feedback.js" "$@"
+      return
+      ;;
+    veritas-governance)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" veritas-governance "$@"
+      return
+      ;;
+    */scripts/validate-hook-influence-cases.js|scripts/validate-hook-influence-cases.js)
+      shift
+      flow_agents_build_ts || return
+      node "$FLOW_AGENTS_EVAL_ROOT/build/src/cli.js" validate-hook-influence "$@"
+      return
+      ;;
+  esac
+  echo "flow_agents_node: no TypeScript adapter registered for $1" >&2
+  return 64
+}