@kontourai/flow-agents 0.1.1

package/evals/integration/test_usage_feedback_import.sh

@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+# test_usage_feedback_import.sh - Layer 2: Usage feedback Codex import validation
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+USAGE_FEEDBACK="$ROOT_DIR/scripts/usage-feedback.js"
+FIXTURE_FULL="$ROOT_DIR/evals/fixtures/usage-feedback/sample-full.jsonl"
+TMPDIR_EVAL=$(mktemp -d /tmp/eval-usage-feedback-import.XXXXXX)
+pass=0; fail=0
+
+cleanup() { rm -rf "$TMPDIR_EVAL"; }
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+_sessions_file() {
+  local dir="$1"
+  if [[ -f "$dir/sessions.jsonl" ]]; then
+    echo "$dir/sessions.jsonl"
+  elif [[ -f "$dir/normalized-sessions.jsonl" ]]; then
+    echo "$dir/normalized-sessions.jsonl"
+  else
+    find "$dir" -maxdepth 2 -type f \( -name 'sessions.jsonl' -o -name 'normalized-sessions.jsonl' \) 2>/dev/null | head -1
+  fi
+}
+
+echo "=== Layer 2: Usage Feedback Runtime Import Validation ==="
+echo ""
+
+echo "--- Script Existence ---"
+if [[ -f "$USAGE_FEEDBACK" ]]; then
+  _pass "usage-feedback.js exists"
+else
+  _fail "usage-feedback.js not found at $USAGE_FEEDBACK"
+  echo ""
+  echo "Result: $pass passed, $fail failed"
+  exit 1
+fi
+
+echo ""
+echo "--- Fixtures ---"
+if [[ -f "$FIXTURE_FULL" ]]; then
+  _pass "sample Codex full.jsonl fixture exists"
+else
+  _fail "sample Codex full.jsonl fixture missing"
+fi
+
+src_claude="$TMPDIR_EVAL/src-claude"
+dst_claude="$TMPDIR_EVAL/dst-claude"
+mkdir -p "$src_claude" "$dst_claude"
+cat > "$src_claude/full.jsonl" <<'JSONL'
+{"schema_version":"0.3.0","timestamp":"2026-05-04T12:00:00Z","session_id":"claude-session-1","event_id":"evt-1","event_type":"turn.user","agent":{"name":"dev","runtime":"claude-code"},"turn":{"prompt_text":"hello","prompt_length":5}}
+{"schema_version":"0.3.0","timestamp":"2026-05-04T12:00:05Z","session_id":"claude-session-1","event_id":"evt-2","event_type":"tool.invoke","agent":{"name":"dev","runtime":"claude-code"},"tool":{"name":"Agent","normalized_name":"delegate to a specialist agent","input":{"subagent_type":"tool-planner"}}}
+{"schema_version":"0.3.0","timestamp":"2026-05-04T12:00:05Z","session_id":"claude-session-1","event_id":"evt-2-delegate","event_type":"agent.delegate","agent":{"name":"dev","runtime":"claude-code"},"delegation":{"targets":["tool-planner"]}}
+{"schema_version":"0.3.0","timestamp":"2026-05-04T12:01:00Z","session_id":"claude-session-1","event_id":"evt-3","event_type":"session.usage","agent":{"name":"dev","runtime":"claude-code"},"usage":{"model":"opus","duration_s":60,"tool_invocations":1,"delegations":1,"input_tokens":null,"output_tokens":null,"estimated_cost_usd":null}}
+JSONL
+
+src="$TMPDIR_EVAL/src"
+dst_a="$TMPDIR_EVAL/dst-a"
+dst_b="$TMPDIR_EVAL/dst-b"
+mkdir -p "$src" "$dst_a" "$dst_b"
+cp "$FIXTURE_FULL" "$src/full.jsonl"
+
+echo ""
+echo "--- Import Command ---"
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-telemetry-dir "$src" \
+  --telemetry-dir "$dst_a" \
+  --source-id "repo-a" \
+  --repo "repo-a" \
+  --repo-root "/tmp/repo-a" \
+  --profile-id "codex-default" \
+  --prompt-id "deliver-v1" \
+  --skill-id "deliver" >/dev/null 2>"$TMPDIR_EVAL/import-a.err"; then
+  _pass "import-codex imports from input telemetry dir"
+else
+  _fail "import-codex failed for input dir: $(cat "$TMPDIR_EVAL/import-a.err" 2>/dev/null)"
+fi
+
+sessions_a="$(_sessions_file "$dst_a")"
+if [[ -n "$sessions_a" && -f "$sessions_a" ]]; then
+  _pass "import-codex writes normalized session data"
+else
+  _fail "import-codex did not write sessions.jsonl or normalized-sessions.jsonl"
+fi
+
+session_count=$(jq -s 'length' "$sessions_a" 2>/dev/null)
+source_id=$(jq -r 'select(.session_id == "codex-session-1") | .source_id' "$sessions_a" 2>/dev/null | head -1)
+repo=$(jq -r 'select(.session_id == "codex-session-1") | .repo' "$sessions_a" 2>/dev/null | head -1)
+profile=$(jq -r 'select(.session_id == "codex-session-1") | .profile_id' "$sessions_a" 2>/dev/null | head -1)
+prompt=$(jq -r 'select(.session_id == "codex-session-1") | .prompt_id' "$sessions_a" 2>/dev/null | head -1)
+skill=$(jq -r 'select(.session_id == "codex-session-1") | .skill_ids[0]' "$sessions_a" 2>/dev/null | head -1)
+if [[ "$session_count" -ge 2 && "$source_id" == "repo-a" && "$repo" == "repo-a" && "$profile" == "codex-default" && "$prompt" == "deliver-v1" && "$skill" == "deliver" ]]; then
+  _pass "normalized sessions include source/repo/profile/prompt/skill identifiers"
+else
+  _fail "normalized identifiers mismatch: count='$session_count' source='$source_id' repo='$repo' profile='$profile' prompt='$prompt' skill='$skill'"
+fi
+
+turns=$(jq -r 'select(.session_id == "codex-session-1") | .turns' "$sessions_a" 2>/dev/null | head -1)
+tools=$(jq -r 'select(.session_id == "codex-session-1") | .tool_invocations' "$sessions_a" 2>/dev/null | head -1)
+delegations=$(jq -r 'select(.session_id == "codex-session-1") | .delegations' "$sessions_a" 2>/dev/null | head -1)
+permissions=$(jq -r 'select(.session_id == "codex-session-1") | .permission_requests' "$sessions_a" 2>/dev/null | head -1)
+if [[ "$turns" == "1" && "$tools" == "2" && "$delegations" == "1" && "$permissions" == "1" ]]; then
+  _pass "normalized sessions preserve Codex usage counts"
+else
+  _fail "usage counts mismatch: turns='$turns' tools='$tools' delegations='$delegations' permissions='$permissions'"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" import-telemetry \
+  --runtime claude-code \
+  --input-telemetry-dir "$src_claude" \
+  --telemetry-dir "$dst_claude" \
+  --source-id "repo-claude" \
+  --repo "repo-claude" \
+  --profile-id "claude-dev" \
+  --prompt-id "deliver-v1" \
+  --skill-id "deliver" >/dev/null 2>"$TMPDIR_EVAL/import-claude.err"; then
+  _pass "import-telemetry imports Claude Code full.jsonl"
+else
+  _fail "import-telemetry failed for Claude Code: $(cat "$TMPDIR_EVAL/import-claude.err" 2>/dev/null)"
+fi
+
+sessions_claude="$(_sessions_file "$dst_claude")"
+claude_runtime=$(jq -r 'select(.session_id == "claude-session-1") | .runtime' "$sessions_claude" 2>/dev/null | head -1)
+claude_tools=$(jq -r 'select(.session_id == "claude-session-1") | .tool_invocations' "$sessions_claude" 2>/dev/null | head -1)
+claude_delegations=$(jq -r 'select(.session_id == "claude-session-1") | .delegations' "$sessions_claude" 2>/dev/null | head -1)
+if [[ "$claude_runtime" == "claude-code" && "$claude_tools" == "1" && "$claude_delegations" == "1" ]]; then
+  _pass "normalized sessions preserve Claude Code runtime and usage counts"
+else
+  _fail "Claude import mismatch: runtime='$claude_runtime' tools='$claude_tools' delegations='$claude_delegations'"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-full-jsonl "$src/full.jsonl" \
+  --telemetry-dir "$dst_b" \
+  --source-id "repo-b" \
+  --repo "repo-b" \
+  --profile-id "codex-experimental" \
+  --prompt-id "deliver-v2" \
+  --skill-id "deliver" >/dev/null 2>"$TMPDIR_EVAL/import-b.err"; then
+  _pass "import-codex imports from explicit full.jsonl"
+else
+  _fail "import-codex failed for explicit full.jsonl: $(cat "$TMPDIR_EVAL/import-b.err" 2>/dev/null)"
+fi
+
+sessions_b="$(_sessions_file "$dst_b")"
+source_b=$(jq -r 'select(.session_id == "codex-session-1") | .source_id' "$sessions_b" 2>/dev/null | head -1)
+if [[ "$source_b" == "repo-b" ]]; then
+  _pass "same fixture can import with a distinct source_id"
+else
+  _fail "second import source_id mismatch: '$source_b'"
+fi
+
+dst_fallback="$TMPDIR_EVAL/dst-fallback"
+mkdir -p "$dst_fallback"
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-full-jsonl "$src/full.jsonl" \
+  --telemetry-dir "$dst_fallback" >/dev/null 2>"$TMPDIR_EVAL/import-fallback.err"; then
+  sessions_fallback="$(_sessions_file "$dst_fallback")"
+  source_fallback=$(jq -r 'select(.session_id == "codex-session-1") | .source_id' "$sessions_fallback" 2>/dev/null | head -1)
+  if [[ "$source_fallback" == "flow-agents" ]]; then
+    _pass "import-codex preserves raw repo source fallback when source-id is omitted"
+  else
+    _fail "omitted source-id fallback mismatch: '$source_fallback'"
+  fi
+else
+  _fail "import-codex failed without source-id: $(cat "$TMPDIR_EVAL/import-fallback.err" 2>/dev/null)"
+fi
+
+src_no_metadata="$TMPDIR_EVAL/src-no-metadata"
+dst_no_metadata="$TMPDIR_EVAL/dst-no-metadata"
+mkdir -p "$src_no_metadata" "$dst_no_metadata"
+cat > "$src_no_metadata/full.jsonl" <<'JSONL'
+{"session_id":"no-metadata-session","event_type":"turn.user","timestamp":"2026-05-04T12:00:00Z"}
+JSONL
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-telemetry-dir "$src_no_metadata" \
+  --telemetry-dir "$dst_no_metadata" >/dev/null 2>"$TMPDIR_EVAL/import-no-metadata.err"; then
+  sessions_no_metadata="$(_sessions_file "$dst_no_metadata")"
+  source_no_metadata=$(jq -r 'select(.session_id == "no-metadata-session") | .source_id' "$sessions_no_metadata" 2>/dev/null | head -1)
+  if [[ "$source_no_metadata" == "src-no-metadata" ]]; then
+    _pass "import-codex uses input telemetry dir name when source metadata is absent"
+  else
+    _fail "input telemetry dir source fallback mismatch: '$source_no_metadata'"
+  fi
+else
+  _fail "import-codex failed for metadata-free input dir: $(cat "$TMPDIR_EVAL/import-no-metadata.err" 2>/dev/null)"
+fi
+
+ln -s "$TMPDIR_EVAL/symlink-target" "$TMPDIR_EVAL/symlink-dst"
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-full-jsonl "$src/full.jsonl" \
+  --telemetry-dir "$TMPDIR_EVAL/symlink-dst" >/dev/null 2>"$TMPDIR_EVAL/import-symlink.err"; then
+  _fail "import-codex accepted symlinked target telemetry dir"
+else
+  _pass "import-codex rejects symlinked target telemetry dir"
+fi
+
+mkdir -p "$TMPDIR_EVAL/import-intermediate-target"
+ln -s "$TMPDIR_EVAL/import-intermediate-target" "$TMPDIR_EVAL/import-intermediate-link"
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-full-jsonl "$src/full.jsonl" \
+  --telemetry-dir "$TMPDIR_EVAL/import-intermediate-link/nested" >/dev/null 2>"$TMPDIR_EVAL/import-symlink-parent.err"; then
+  _fail "import-codex accepted target telemetry dir with symlinked parent"
+else
+  if [[ ! -e "$TMPDIR_EVAL/import-intermediate-target/nested/normalized-sessions.jsonl" ]]; then
+    _pass "import-codex rejects symlinked target telemetry parent before creating nested dirs"
+  else
+    _fail "import-codex wrote through symlinked target telemetry parent"
+  fi
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" import-codex \
+  --input-telemetry-dir "$TMPDIR_EVAL/missing" \
+  --telemetry-dir "$TMPDIR_EVAL/missing-dst" \
+  --source-id "missing" \
+  --repo "missing" >/dev/null 2>"$TMPDIR_EVAL/missing.err"; then
+  _fail "import-codex accepted missing input telemetry"
+else
+  _pass "import-codex rejects missing input telemetry"
+fi
+
+echo ""
+echo "Result: $pass passed, $fail failed"
+[[ $fail -eq 0 ]]

package/evals/integration/test_usage_feedback_outcomes.sh

@@ -0,0 +1,165 @@
+#!/usr/bin/env bash
+# test_usage_feedback_outcomes.sh - Layer 2: Usage feedback outcome validation
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+USAGE_FEEDBACK="$ROOT_DIR/scripts/usage-feedback.js"
+TMPDIR_EVAL=$(mktemp -d /tmp/eval-usage-feedback-outcomes.XXXXXX)
+pass=0; fail=0
+
+cleanup() { rm -rf "$TMPDIR_EVAL"; }
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+echo "=== Layer 2: Usage Feedback Outcome Validation ==="
+echo ""
+
+echo "--- Script Existence ---"
+if [[ -f "$USAGE_FEEDBACK" ]]; then
+  _pass "usage-feedback.js exists"
+else
+  _fail "usage-feedback.js not found at $USAGE_FEEDBACK"
+  echo ""
+  echo "Result: $pass passed, $fail failed"
+  exit 1
+fi
+
+echo ""
+echo "--- Outcome Recording ---"
+if TELEMETRY_DATA_DIR="$TMPDIR_EVAL" flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --session-id "session-success" \
+  --runtime "codex" \
+  --repo "flow-agents" \
+  --agent "dev" \
+  --profile-id "codex-default" \
+  --prompt-id "deliver-v1" \
+  --skill-id "deliver" \
+  --result "success" \
+  --quality-score 5 \
+  --task-type "deliver" \
+  --task-slug "usage-feedback-success" \
+  --human-minutes-saved 12 \
+  --evidence ".flow-agents/agent-usage-feedback-loop/agent-usage-feedback-loop--deliver.md" >/dev/null 2>"$TMPDIR_EVAL/success.err"; then
+  _pass "record-outcome accepts success with profile/prompt/skill ids"
+else
+  _fail "record-outcome rejected success: $(cat "$TMPDIR_EVAL/success.err" 2>/dev/null)"
+fi
+
+if TELEMETRY_DATA_DIR="$TMPDIR_EVAL" flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --session-id "session-failure" \
+  --runtime "codex" \
+  --repo "flow-agents-docs" \
+  --agent "dev" \
+  --profile-id "codex-experimental" \
+  --prompt-id "deliver-v2" \
+  --skill-id "deliver" \
+  --skill-id "verify-work" \
+  --result "failure" \
+  --quality-score 2 \
+  --task-type "verify" \
+  --task-slug "usage-feedback-failure" \
+  --rework-required \
+  --notes "Fixture failure" >/dev/null 2>"$TMPDIR_EVAL/failure.err"; then
+  _pass "record-outcome accepts failure with multiple skill ids"
+else
+  _fail "record-outcome rejected failure: $(cat "$TMPDIR_EVAL/failure.err" 2>/dev/null)"
+fi
+
+OUTCOMES="$TMPDIR_EVAL/outcomes.jsonl"
+line_count=$(wc -l < "$OUTCOMES" 2>/dev/null | tr -d ' ')
+if [[ "$line_count" == "2" ]]; then
+  _pass "record-outcome appends two outcome records"
+else
+  _fail "expected 2 outcome records, found ${line_count:-0}"
+fi
+
+success_profile=$(jq -r 'select(.session_id == "session-success") | .profile_id' "$OUTCOMES" 2>/dev/null)
+success_prompt=$(jq -r 'select(.session_id == "session-success") | .prompt_id' "$OUTCOMES" 2>/dev/null)
+success_skill=$(jq -r 'select(.session_id == "session-success") | .skill_ids[0]' "$OUTCOMES" 2>/dev/null)
+if [[ "$success_profile" == "codex-default" && "$success_prompt" == "deliver-v1" && "$success_skill" == "deliver" ]]; then
+  _pass "success outcome preserves profile, prompt, and skill ids"
+else
+  _fail "success identifiers mismatch: profile='$success_profile' prompt='$success_prompt' skill='$success_skill'"
+fi
+
+failure_rework=$(jq -r 'select(.session_id == "session-failure") | .rework_required' "$OUTCOMES" 2>/dev/null)
+failure_skill_count=$(jq -r 'select(.session_id == "session-failure") | .skill_ids | length' "$OUTCOMES" 2>/dev/null)
+if [[ "$failure_rework" == "true" && "$failure_skill_count" == "2" ]]; then
+  _pass "failure outcome preserves rework flag and multiple skill ids"
+else
+  _fail "failure fields mismatch: rework='$failure_rework' skill_count='$failure_skill_count'"
+fi
+
+before_invalid=$(wc -l < "$OUTCOMES" 2>/dev/null | tr -d ' ')
+if TELEMETRY_DATA_DIR="$TMPDIR_EVAL" flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --session-id "session-invalid" \
+  --runtime "codex" \
+  --repo "flow-agents" \
+  --agent "dev" \
+  --profile-id "codex-default" \
+  --prompt-id "deliver-v1" \
+  --skill-id "deliver" \
+  --result "excellent" >/dev/null 2>"$TMPDIR_EVAL/invalid-result.err"; then
+  _fail "record-outcome accepted invalid result"
+else
+  after_invalid=$(wc -l < "$OUTCOMES" 2>/dev/null | tr -d ' ')
+  if [[ "$after_invalid" == "$before_invalid" ]]; then
+    _pass "record-outcome rejects invalid result without appending"
+  else
+    _fail "invalid result changed outcomes.jsonl line count from $before_invalid to $after_invalid"
+  fi
+fi
+
+if TELEMETRY_DATA_DIR="$TMPDIR_EVAL" flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --runtime "codex" \
+  --repo "flow-agents" \
+  --agent "dev" \
+  --result "success" >/dev/null 2>"$TMPDIR_EVAL/missing-session.err"; then
+  _fail "record-outcome accepted missing session_id"
+else
+  _pass "record-outcome rejects missing session_id"
+fi
+
+ln -s "$TMPDIR_EVAL/symlink-target" "$TMPDIR_EVAL/symlink-telemetry"
+if flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --telemetry-dir "$TMPDIR_EVAL/symlink-telemetry" \
+  --session-id "session-symlink-dir" \
+  --result "success" >/dev/null 2>"$TMPDIR_EVAL/symlink-dir.err"; then
+  _fail "record-outcome accepted symlinked telemetry dir"
+else
+  _pass "record-outcome rejects symlinked telemetry dir"
+fi
+
+mkdir -p "$TMPDIR_EVAL/intermediate-target"
+ln -s "$TMPDIR_EVAL/intermediate-target" "$TMPDIR_EVAL/intermediate-link"
+if flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --telemetry-dir "$TMPDIR_EVAL/intermediate-link/nested" \
+  --session-id "session-symlink-parent" \
+  --result "success" >/dev/null 2>"$TMPDIR_EVAL/symlink-parent.err"; then
+  _fail "record-outcome accepted telemetry dir with symlinked parent"
+else
+  if [[ ! -e "$TMPDIR_EVAL/intermediate-target/nested/outcomes.jsonl" ]]; then
+    _pass "record-outcome rejects symlinked telemetry parent before creating nested dirs"
+  else
+    _fail "record-outcome wrote through symlinked telemetry parent"
+  fi
+fi
+
+target_file_dir="$TMPDIR_EVAL/symlink-file-telemetry"
+mkdir -p "$target_file_dir"
+ln -s "$TMPDIR_EVAL/symlink-outcomes-target.jsonl" "$target_file_dir/outcomes.jsonl"
+if flow_agents_node "$USAGE_FEEDBACK" record-outcome \
+  --telemetry-dir "$target_file_dir" \
+  --session-id "session-symlink-file" \
+  --result "success" >/dev/null 2>"$TMPDIR_EVAL/symlink-file.err"; then
+  _fail "record-outcome accepted symlinked outcomes target"
+else
+  _pass "record-outcome rejects symlinked outcomes target"
+fi
+
+echo ""
+echo "Result: $pass passed, $fail failed"
+[[ $fail -eq 0 ]]

package/evals/integration/test_usage_feedback_report.sh

@@ -0,0 +1,263 @@
+#!/usr/bin/env bash
+# test_usage_feedback_report.sh - Layer 2: Usage feedback report validation
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+USAGE_FEEDBACK="$ROOT_DIR/scripts/usage-feedback.js"
+FIXTURE_DIR="$ROOT_DIR/evals/fixtures/usage-feedback"
+TMPDIR_EVAL=$(mktemp -d /tmp/eval-usage-feedback-report.XXXXXX)
+pass=0; fail=0
+
+cleanup() { rm -rf "$TMPDIR_EVAL"; }
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+echo "=== Layer 2: Usage Feedback Report Validation ==="
+echo ""
+
+echo "--- Script Existence ---"
+if [[ -f "$USAGE_FEEDBACK" ]]; then
+  _pass "usage-feedback.js exists"
+else
+  _fail "usage-feedback.js not found at $USAGE_FEEDBACK"
+  echo ""
+  echo "Result: $pass passed, $fail failed"
+  exit 1
+fi
+
+echo ""
+tmp_a="$TMPDIR_EVAL/repo-a"
+tmp_b="$TMPDIR_EVAL/repo-b"
+mkdir -p "$tmp_a" "$tmp_b"
+cp "$FIXTURE_DIR/sample-full.jsonl" "$tmp_a/full.jsonl"
+cp "$FIXTURE_DIR/sample-outcomes.jsonl" "$tmp_a/outcomes.jsonl"
+cp "$FIXTURE_DIR/sample-full.jsonl" "$tmp_b/full.jsonl"
+cat > "$tmp_b/outcomes.jsonl" <<'JSONL'
+{"schema_version":"1","outcome_id":"outcome-2","recorded_at":"2026-05-04T11:30:00Z","session_id":"codex-session-2","runtime":"codex","repo":"repo-b","agent":"dev","profile_id":"codex-experimental","prompt_id":"deliver-v2","prompt_variant":"concise","skill_ids":["deliver","verify-work"],"task_type":"verify","task_slug":"usage-feedback-report","result":"failure","quality_score":2,"human_minutes_saved":0,"rework_required":true,"notes":"Fixture failure outcome","evidence":["evals/integration/test_usage_feedback_report.sh"]}
+JSONL
+
+echo "--- JSON Report ---"
+json_report="$TMPDIR_EVAL/report.json"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --telemetry-dir "$tmp_b" \
+  --format json \
+  --group-by repo >"$json_report" 2>"$TMPDIR_EVAL/report-json.err"; then
+  _pass "report emits JSON for multiple telemetry dirs"
+else
+  _fail "JSON report failed: $(cat "$TMPDIR_EVAL/report-json.err" 2>/dev/null)"
+fi
+
+if jq -e '.summary.sessions >= 4 and .summary.sessions_with_outcomes >= 2 and (.summary.success_rate != null) and (.sources | length >= 2)' "$json_report" >/dev/null 2>&1; then
+  _pass "JSON report includes summary sessions, outcomes, success rate, and sources"
+else
+  _fail "JSON report missing expected summary/source fields"
+fi
+
+if jq -e '.groups[]? | select((.key == "flow-agents") or (.group == "flow-agents") or (.name == "flow-agents"))' "$json_report" >/dev/null 2>&1; then
+  _pass "JSON report groups by repo"
+else
+  _fail "JSON report did not include repo group"
+fi
+
+if jq -e '.summary.sessions > .summary.sessions_with_outcomes' "$json_report" >/dev/null 2>&1; then
+  _pass "report includes sessions without outcomes in usage totals"
+else
+  _fail "report did not distinguish sessions without outcomes"
+fi
+
+echo ""
+echo "--- Markdown Report ---"
+markdown_report="$tmp_a/reports/usage.md"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --group-by profile_id \
+  --output "$markdown_report" >/dev/null 2>"$TMPDIR_EVAL/report-md.err"; then
+  _pass "report writes Markdown output file"
+else
+  _fail "Markdown report failed: $(cat "$TMPDIR_EVAL/report-md.err" 2>/dev/null)"
+fi
+
+if [[ -f "$markdown_report" ]]; then
+  _pass "Markdown report output file exists"
+else
+  _fail "Markdown report output file missing"
+fi
+
+if grep -q "# Agent Usage Feedback Report" "$markdown_report" && \
+   grep -q "Success rate" "$markdown_report" && \
+   grep -q "Avg tool invocations" "$markdown_report" && \
+   grep -q "Rework rate" "$markdown_report" && \
+   grep -q "codex-default" "$markdown_report"; then
+  _pass "Markdown report includes required headings, metrics, and profile group"
+else
+  _fail "Markdown report missing required content"
+fi
+
+relative_report="$tmp_a/reports/relative.md"
+if (cd "$TMPDIR_EVAL" && flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --group-by profile_id \
+  --output reports/relative.md >/dev/null 2>"$TMPDIR_EVAL/report-relative.err") && [[ -f "$relative_report" ]]; then
+  _pass "report writes reports/name.md relative to telemetry reports directory"
+else
+  _fail "relative reports/name.md output failed: $(cat "$TMPDIR_EVAL/report-relative.err" 2>/dev/null)"
+fi
+
+nested_guard_report="$tmp_a/reports/usage-feedback.md"
+if (cd "$TMPDIR_EVAL" && flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --group-by profile_id \
+  --output "$(basename "$tmp_a")/reports/usage-feedback.md" >/dev/null 2>"$TMPDIR_EVAL/report-nested-guard.err") && \
+   [[ -f "$nested_guard_report" && ! -e "$tmp_a/reports/$(basename "$tmp_a")/reports/usage-feedback.md" ]]; then
+  _pass "report prevents nested telemetry reports duplication for relative output"
+else
+  _fail "nested report output guard failed: $(cat "$TMPDIR_EVAL/report-nested-guard.err" 2>/dev/null)"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --group-by profile_id \
+  --output "$markdown_report" >/dev/null 2>"$TMPDIR_EVAL/report-overwrite.err"; then
+  _fail "report overwrote existing output without --force"
+else
+  _pass "report rejects existing output without --force"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --group-by profile_id \
+  --output "$markdown_report" \
+  --force >/dev/null 2>"$TMPDIR_EVAL/report-force.err"; then
+  _pass "report overwrites existing output with --force"
+else
+  _fail "report --force failed: $(cat "$TMPDIR_EVAL/report-force.err" 2>/dev/null)"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --output "$TMPDIR_EVAL/outside.md" >/dev/null 2>"$TMPDIR_EVAL/report-outside.err"; then
+  _fail "report accepted output outside telemetry reports directory"
+else
+  _pass "report rejects output outside telemetry reports directory"
+fi
+
+ln -s "$TMPDIR_EVAL/symlink-report-target.md" "$tmp_a/reports/symlink.md"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --output "$tmp_a/reports/symlink.md" \
+  --force >/dev/null 2>"$TMPDIR_EVAL/report-symlink.err"; then
+  _fail "report accepted symlinked output target"
+else
+  _pass "report rejects symlinked output target"
+fi
+
+tmp_symlink_reports="$TMPDIR_EVAL/symlink-reports"
+mkdir -p "$tmp_symlink_reports"
+cp "$FIXTURE_DIR/sample-full.jsonl" "$tmp_symlink_reports/full.jsonl"
+ln -s "$TMPDIR_EVAL/report-parent-target" "$tmp_symlink_reports/reports"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_symlink_reports" \
+  --output usage.md >/dev/null 2>"$TMPDIR_EVAL/report-symlink-parent.err"; then
+  _fail "report accepted symlinked reports directory"
+else
+  _pass "report rejects symlinked reports directory"
+fi
+
+mkdir -p "$TMPDIR_EVAL/report-intermediate-target"
+ln -s "$TMPDIR_EVAL/report-intermediate-target" "$TMPDIR_EVAL/report-intermediate-link"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$TMPDIR_EVAL/report-intermediate-link/nested" \
+  --output usage.md >/dev/null 2>"$TMPDIR_EVAL/report-symlink-telemetry-parent.err"; then
+  _fail "report accepted telemetry dir with symlinked parent"
+else
+  if [[ ! -e "$TMPDIR_EVAL/report-intermediate-target/nested/reports/usage.md" ]]; then
+    _pass "report rejects symlinked telemetry parent before creating report dirs"
+  else
+    _fail "report wrote through symlinked telemetry parent"
+  fi
+fi
+
+tmp_raw="$TMPDIR_EVAL/raw-source-name"
+mkdir -p "$tmp_raw"
+cat > "$tmp_raw/full.jsonl" <<'JSONL'
+{"session_id":"raw-session","event_type":"turn.user","timestamp":"2026-05-04T12:00:00Z"}
+JSONL
+raw_report="$TMPDIR_EVAL/raw-report.json"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_raw" \
+  --format json >"$raw_report" 2>"$TMPDIR_EVAL/report-raw.err" && \
+   jq -e '.sources == ["raw-source-name"]' "$raw_report" >/dev/null 2>&1; then
+  _pass "raw telemetry without source metadata groups by telemetry directory name"
+else
+  _fail "raw telemetry source fallback failed: $(cat "$TMPDIR_EVAL/report-raw.err" 2>/dev/null)"
+fi
+
+tmp_escape="$TMPDIR_EVAL/escape-source"
+mkdir -p "$tmp_escape"
+cat > "$tmp_escape/normalized-sessions.jsonl" <<'JSONL'
+{"schema_version":"1","source_id":"escape-source","runtime":"codex","session_id":"escape-session","profile_id":"alpha|beta\nbreak <tag> & value","skill_ids":[],"turns":0,"tool_invocations":0,"delegations":0,"permission_requests":0}
+JSONL
+escape_report="$TMPDIR_EVAL/escape.md"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_escape" \
+  --group-by profile_id >"$escape_report" 2>"$TMPDIR_EVAL/report-escape.err" && \
+   grep -q 'alpha\\|beta break &lt;tag&gt; &amp; value' "$escape_report"; then
+  _pass "Markdown report escapes table labels, HTML chars, and strips newlines"
+else
+  _fail "Markdown report label escaping failed: $(cat "$TMPDIR_EVAL/report-escape.err" 2>/dev/null)"
+fi
+
+escape_html="$tmp_escape/reports/escape.html"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_escape" \
+  --group-by profile_id \
+  --format html \
+  --output "$escape_html" >"$TMPDIR_EVAL/report-html-escape.out" 2>"$TMPDIR_EVAL/report-html-escape.err" && \
+   grep -q '&lt;tag&gt; &amp; value' "$escape_html" && \
+   ! grep -q '<tag>' "$escape_html"; then
+  _pass "HTML report escapes local telemetry labels"
+else
+  _fail "HTML report escaping failed: $(cat "$TMPDIR_EVAL/report-html-escape.err" 2>/dev/null)"
+fi
+
+echo ""
+echo "--- Fixture Report Smoke ---"
+fixture_report="$TMPDIR_EVAL/fixture-runtime.md"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$tmp_a" \
+  --group-by runtime >"$fixture_report" 2>"$TMPDIR_EVAL/fixture.err"; then
+  _pass "report works against copied fixture telemetry"
+else
+  _fail "fixture report failed: $(cat "$TMPDIR_EVAL/fixture.err" 2>/dev/null)"
+fi
+
+direct_fixture_report="$TMPDIR_EVAL/direct-fixture-repo.md"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$FIXTURE_DIR" \
+  --group-by repo >"$direct_fixture_report" 2>"$TMPDIR_EVAL/direct-fixture-repo.err" && \
+   grep -q "Sessions: 2" "$direct_fixture_report" && \
+   grep -q "flow-agents-docs" "$direct_fixture_report"; then
+  _pass "report reads sample fixture names directly for repo groups"
+else
+  _fail "direct fixture repo report failed: $(cat "$TMPDIR_EVAL/direct-fixture-repo.err" 2>/dev/null)"
+fi
+
+direct_profile_report="$TMPDIR_EVAL/direct-fixture-profile.md"
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$FIXTURE_DIR" \
+  --group-by profile_id >"$direct_profile_report" 2>"$TMPDIR_EVAL/direct-fixture-profile.err" && \
+   grep -q "Sessions: 2" "$direct_profile_report" && \
+   grep -q "codex-default" "$direct_profile_report" && \
+   grep -q "codex-experimental" "$direct_profile_report"; then
+  _pass "report reads sample fixture names directly for profile groups"
+else
+  _fail "direct fixture profile report failed: $(cat "$TMPDIR_EVAL/direct-fixture-profile.err" 2>/dev/null)"
+fi
+
+echo ""
+echo "Result: $pass passed, $fail failed"
+[[ $fail -eq 0 ]]