@kontourai/flow-agents 0.1.1

package/evals/integration/test_telemetry_doctor.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+FLOW_AGENTS_CLI="${FLOW_AGENTS_CLI:-node}"
+FLOW_AGENTS_CLI_ARGS=()
+if [[ "$FLOW_AGENTS_CLI" == "node" ]]; then
+  FLOW_AGENTS_CLI_ARGS=("$ROOT_DIR/build/src/cli.js")
+fi
+TMPDIR_EVAL="$(mktemp -d /tmp/flow-agents-telemetry-doctor.XXXXXX)"
+pass=0
+fail=0
+
+cleanup() {
+  rm -rf "$TMPDIR_EVAL"
+}
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+echo "=== Telemetry Doctor Integration Test ==="
+echo ""
+
+DEST="$TMPDIR_EVAL/install"
+mkdir -p "$DEST/scripts/telemetry"
+cp "$ROOT_DIR/scripts/telemetry/telemetry.conf" "$DEST/scripts/telemetry/telemetry.conf"
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless >"$TMPDIR_EVAL/local.json"; then
+  if jq -e '
+    .ok == true and
+    .telemetry.configExists == true and
+    (.telemetry.activeSinks | index("local-files")) and
+    .console.sink == "local-only" and
+    .console.reachability.checked == false
+  ' "$TMPDIR_EVAL/local.json" >/dev/null; then
+    _pass "doctor reports local-only telemetry as JSON"
+  else
+    _fail "doctor local-only JSON did not match expected shape"
+  fi
+else
+  _fail "doctor failed for local-only telemetry config"
+fi
+
+cat >> "$DEST/scripts/telemetry/telemetry.conf" <<'CONF'
+console_telemetry_url=http://127.0.0.1:9
+console_tenant_id=tenant-a
+CONF
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless --timeout-ms 250 >"$TMPDIR_EVAL/console.json"; then
+  _fail "doctor returned success for unreachable configured Console"
+else
+  if jq -e '
+    .ok == false and
+    (.telemetry.activeSinks | index("console")) and
+    .console.sink == "console" and
+    .console.endpointAllowed == true and
+    .console.tenantConfigured == true and
+    .console.reachability.checked == true and
+    .console.reachability.ok == false
+  ' "$TMPDIR_EVAL/console.json" >/dev/null; then
+    _pass "doctor reports unreachable configured Console in JSON"
+  else
+    _fail "doctor unreachable Console JSON did not match expected shape"
+  fi
+fi
+
+cp "$ROOT_DIR/scripts/telemetry/telemetry.conf" "$DEST/scripts/telemetry/telemetry.conf"
+cat >> "$DEST/scripts/telemetry/telemetry.conf" <<'CONF'
+console_telemetry_url=http://example.test
+CONF
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless --timeout-ms 250 >"$TMPDIR_EVAL/unsafe.json"; then
+  _fail "doctor returned success for unsafe Console URL"
+else
+  if jq -e '
+    .ok == false and
+    .console.sink == "console" and
+    .console.endpointAllowed == false and
+    .console.reachability.checked == false and
+    (.warnings | length > 0)
+  ' "$TMPDIR_EVAL/unsafe.json" >/dev/null; then
+    _pass "doctor rejects unsafe non-local Console URL before reachability"
+  else
+    _fail "doctor unsafe Console JSON did not match expected shape"
+  fi
+fi
+
+cat > "$DEST/scripts/telemetry/telemetry.conf" <<'CONF'
+enabled=true
+channels=full,analytics
+console_telemetry_url=https://bad host
+CONF
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless >"$TMPDIR_EVAL/malformed.json"; then
+  _fail "doctor returned success for malformed HTTPS Console URL"
+else
+  if jq -e '
+    .ok == false and
+    .console.endpointAllowed == false and
+    .console.reachability.checked == false and
+    .console.endpointUrl == "[malformed-url]"
+  ' "$TMPDIR_EVAL/malformed.json" >/dev/null; then
+    _pass "doctor emits JSON for malformed HTTPS Console URL"
+  else
+    _fail "doctor malformed HTTPS JSON did not match expected shape"
+  fi
+fi
+
+cat > "$DEST/scripts/telemetry/telemetry.conf" <<'CONF'
+enabled=true
+channels=full,analytics
+console_telemetry_url=https://user:pass@console.example.test/path?token=secret&safe=yes
+CONF
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless >"$TMPDIR_EVAL/redacted.json"; then
+  _fail "doctor returned success for credential-bearing Console URL"
+else
+  if jq -e '
+    .ok == false and
+    .console.endpointAllowed == false and
+    (.console.url | contains("user") | not) and
+    (.console.url | contains("pass") | not) and
+    (.console.url | contains("secret") | not) and
+    (.console.url | contains("token=%5Bredacted%5D"))
+  ' "$TMPDIR_EVAL/redacted.json" >/dev/null; then
+    _pass "doctor redacts credential-bearing Console URLs"
+  else
+    _fail "doctor did not redact credential-bearing Console URLs"
+  fi
+fi
+
+cat > "$DEST/scripts/telemetry/telemetry.conf" <<'CONF'
+enabled=true
+channels=full
+channel.full.endpoint_url=https://user:pass@example.test/path?token=secret&safe=yes
+CONF
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless >"$TMPDIR_EVAL/channel-redacted.json"; then
+  if jq -e '
+    .ok == true and
+    (.telemetry.channels[0].endpointUrl | contains("user") | not) and
+    (.telemetry.channels[0].endpointUrl | contains("pass") | not) and
+    (.telemetry.channels[0].endpointUrl | contains("secret") | not) and
+    (.telemetry.channels[0].endpointUrl | contains("token=%5Bredacted%5D"))
+  ' "$TMPDIR_EVAL/channel-redacted.json" >/dev/null; then
+    _pass "doctor redacts channel endpoint URLs"
+  else
+    _fail "doctor did not redact channel endpoint URLs"
+  fi
+else
+  _fail "doctor failed for channel endpoint redaction case"
+fi
+
+cat > "$DEST/scripts/telemetry/telemetry.conf" <<'CONF'
+enabled=true
+channels=full,analytics
+console_telemetry_url=https://console.example.test
+CONF
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless >"$TMPDIR_EVAL/nonlocal.json"; then
+  _fail "doctor returned success for non-local Console without --allow-network"
+else
+  if jq -e '
+    .ok == false and
+    .console.endpointAllowed == false and
+    .console.reachability.checked == false
+  ' "$TMPDIR_EVAL/nonlocal.json" >/dev/null; then
+    _pass "doctor blocks non-local Console reachability without opt-in"
+  else
+    _fail "doctor non-local block JSON did not match expected shape"
+  fi
+fi
+
+if $FLOW_AGENTS_CLI "${FLOW_AGENTS_CLI_ARGS[@]}" telemetry-doctor --dest "$DEST" --json --headless --allow-network --timeout-ms 50 >"$TMPDIR_EVAL/allow-network.json"; then
+  _fail "doctor returned success for unreachable non-local Console with --allow-network"
+else
+  if jq -e '
+    .ok == false and
+    .console.endpointAllowed == true and
+    .console.reachability.checked == true
+  ' "$TMPDIR_EVAL/allow-network.json" >/dev/null; then
+    _pass "doctor checks non-local Console only with explicit network opt-in"
+  else
+    _fail "doctor allow-network JSON did not match expected shape"
+  fi
+fi
+
+echo ""
+echo "Passed: $pass"
+echo "Failed: $fail"
+
+[[ "$fail" -eq 0 ]]

package/evals/integration/test_usage_feedback_dashboard.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# test_usage_feedback_dashboard.sh - Layer 2: automatic artifact sync + HTML dashboard validation
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+USAGE_FEEDBACK="$ROOT_DIR/scripts/usage-feedback.js"
+TMPDIR_EVAL=$(mktemp -d /tmp/eval-usage-feedback-dashboard.XXXXXX)
+pass=0; fail=0
+
+cleanup() { rm -rf "$TMPDIR_EVAL"; }
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+echo "=== Layer 2: Usage Feedback Dashboard Validation ==="
+echo ""
+
+echo "--- Script Existence ---"
+if [[ -f "$USAGE_FEEDBACK" ]]; then
+  _pass "usage-feedback.js exists"
+else
+  _fail "usage-feedback.js not found at $USAGE_FEEDBACK"
+  echo ""
+  echo "Result: $pass passed, $fail failed"
+  exit 1
+fi
+
+telemetry="$TMPDIR_EVAL/telemetry"
+artifacts="$TMPDIR_EVAL/.flow-agents"
+mkdir -p "$telemetry" "$artifacts/auto-feedback" "$artifacts/open-feedback"
+cat > "$artifacts/auto-feedback/auto-feedback--deliver.md" <<'MARKDOWN'
+# Auto Feedback Delivery
+
+branch: main
+worktree: /tmp/example
+created: 2026-05-04
+status: delivered
+type: deliver
+iteration: 1
+MARKDOWN
+
+cat > "$artifacts/open-feedback/open-feedback--deliver.md" <<'MARKDOWN'
+# Open Feedback Delivery
+
+branch: main
+worktree: /tmp/example
+created: 2026-05-04
+status: verifying
+type: deliver
+iteration: 0
+MARKDOWN
+mkdir -p "$artifacts/html-feedback"
+cat > "$artifacts/html-feedback/html-feedback--deliver.md" <<'MARKDOWN'
+# <script>alert(1)</script>
+
+branch: main
+worktree: /tmp/example
+created: 2026-05-04
+status: delivered
+type: deliver
+iteration: 1
+MARKDOWN
+
+echo ""
+echo "--- Artifact Sync ---"
+if flow_agents_node "$USAGE_FEEDBACK" sync-artifacts \
+  --telemetry-dir "$telemetry" \
+  --artifact-dir "$artifacts" \
+  --repo flow-agents \
+  --profile-id codex-default \
+  --prompt-id deliver-v1 \
+  --skill-id deliver >"$TMPDIR_EVAL/sync.out" 2>"$TMPDIR_EVAL/sync.err"; then
+  _pass "sync-artifacts derives terminal artifact outcomes"
+else
+  _fail "sync-artifacts failed: $(cat "$TMPDIR_EVAL/sync.err" 2>/dev/null)"
+fi
+
+if [[ -f "$telemetry/outcomes.jsonl" ]] && \
+   jq -e 'select(.result == "success" and .task_type == "deliver" and .quality_score == null)' "$telemetry/outcomes.jsonl" >/dev/null 2>&1; then
+  _pass "synced outcome records success without invented quality score"
+else
+  _fail "synced outcome missing expected success/null-quality fields"
+fi
+
+before_count=$(wc -l < "$telemetry/outcomes.jsonl" | tr -d ' ')
+flow_agents_node "$USAGE_FEEDBACK" sync-artifacts \
+  --telemetry-dir "$telemetry" \
+  --artifact-dir "$artifacts" \
+  --repo flow-agents \
+  --profile-id codex-default \
+  --prompt-id deliver-v1 \
+  --skill-id deliver >/dev/null 2>"$TMPDIR_EVAL/sync-second.err"
+after_count=$(wc -l < "$telemetry/outcomes.jsonl" | tr -d ' ')
+if [[ "$before_count" == "$after_count" ]]; then
+  _pass "sync-artifacts is idempotent by artifact outcome id"
+else
+  _fail "sync-artifacts duplicated outcomes on second run"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" sync-artifacts \
+  --telemetry-dir "$telemetry" \
+  --artifact-dir "$artifacts" \
+  --include-open \
+  --repo flow-agents >/dev/null 2>"$TMPDIR_EVAL/sync-open.err" && \
+   jq -e 'select(.result == "not_verified" and .task_slug == "open-feedback-delivery")' "$telemetry/outcomes.jsonl" >/dev/null 2>&1; then
+  _pass "sync-artifacts can include open artifacts as not_verified"
+else
+  _fail "sync-artifacts --include-open did not record open artifact"
+fi
+
+cat >> "$telemetry/outcomes.jsonl" <<'JSONL'
+{"schema_version":"1","outcome_id":"xss-outcome","recorded_at":"2026-05-04T11:30:00Z","session_id":"xss-session","runtime":"codex","repo":"flow-agents","task_type":"deliver","task_slug":"<script>alert(1)</script>","result":"success","quality_score":null,"human_minutes_saved":null,"rework_required":false,"evidence":[]}
+JSONL
+
+echo ""
+echo "--- Dashboard ---"
+dashboard="$telemetry/reports/dashboard.html"
+if flow_agents_node "$USAGE_FEEDBACK" dashboard \
+  --telemetry-dir "$telemetry" \
+  --artifact-dir "$artifacts" \
+  --repo flow-agents \
+  --profile-id codex-default \
+  --prompt-id deliver-v1 \
+  --skill-id deliver \
+  --force >"$TMPDIR_EVAL/dashboard.out" 2>"$TMPDIR_EVAL/dashboard.err"; then
+  _pass "dashboard syncs artifacts and writes HTML"
+else
+  _fail "dashboard failed: $(cat "$TMPDIR_EVAL/dashboard.err" 2>/dev/null)"
+fi
+
+if [[ -f "$dashboard" ]] && \
+   grep -q "<!doctype html>" "$dashboard" && \
+   grep -q "Usage Dashboard" "$dashboard" && \
+   grep -q "What Needs Attention" "$dashboard" && \
+   grep -q "Measurement state" "$dashboard" && \
+   grep -q "Data Coverage" "$dashboard" && \
+   grep -q "Outcome Mix" "$dashboard" && \
+   grep -q "Missing Label Drilldown" "$dashboard" && \
+   grep -q "auto-feedback-delivery" "$dashboard" && \
+   grep -q '&lt;script&gt;alert(1)&lt;/script&gt;' "$dashboard" && \
+   ! grep -q '<script>alert(1)</script>' "$dashboard"; then
+  _pass "dashboard HTML contains expected sections and escapes artifact labels"
+else
+  _fail "dashboard HTML missing expected content or escaping"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" dashboard \
+  --telemetry-dir "$telemetry" \
+  --artifact-dir "$artifacts" >/dev/null 2>"$TMPDIR_EVAL/dashboard-overwrite.err"; then
+  _fail "dashboard overwrote existing output without --force"
+else
+  _pass "dashboard rejects existing output without --force"
+fi
+
+if flow_agents_node "$USAGE_FEEDBACK" report \
+  --telemetry-dir "$telemetry" \
+  --format html \
+  --output reports/report.html >"$TMPDIR_EVAL/report-html.out" 2>"$TMPDIR_EVAL/report-html.err" && \
+   grep -q "Usage Dashboard" "$telemetry/reports/report.html"; then
+  _pass "report --format html writes dashboard-style HTML"
+else
+  _fail "report --format html failed: $(cat "$TMPDIR_EVAL/report-html.err" 2>/dev/null)"
+fi
+
+echo ""
+echo "Result: $pass passed, $fail failed"
+[[ $fail -eq 0 ]]

package/evals/integration/test_usage_feedback_global.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# test_usage_feedback_global.sh - Layer 2: global usage registry/dashboard validation
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+USAGE_FEEDBACK="$ROOT_DIR/scripts/usage-feedback.js"
+TMPDIR_EVAL=$(mktemp -d /tmp/eval-usage-feedback-global.XXXXXX)
+pass=0; fail=0
+
+cleanup() { rm -rf "$TMPDIR_EVAL"; }
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+echo "=== Layer 2: Usage Feedback Global Validation ==="
+echo ""
+
+global="$TMPDIR_EVAL/global"
+repo_a="$TMPDIR_EVAL/repo-a"
+repo_b="$TMPDIR_EVAL/repo-b"
+mkdir -p "$repo_a/.flow-agents/alpha" "$repo_b/.flow-agents/beta"
+
+cat > "$repo_a/.flow-agents/alpha/alpha--deliver.md" <<'MARKDOWN'
+# Deliver Alpha
+
+status: delivered
+type: deliver
+MARKDOWN
+
+cat > "$repo_b/.flow-agents/beta/beta--verify.md" <<'MARKDOWN'
+# Verify Beta
+
+status: failed
+type: verify-work
+MARKDOWN
+
+echo "--- Registration ---"
+if flow_agents_node "$USAGE_FEEDBACK" register-project \
+  --global-dir "$global" \
+  --repo-root "$repo_a" \
+  --name alpha \
+  --profile-id codex-default \
+  --prompt-id deliver-v1 \
+  --skill-id deliver >"$TMPDIR_EVAL/register.out" 2>"$TMPDIR_EVAL/register.err"; then
+  _pass "register-project writes global project registry"
+else
+  _fail "register-project failed: $(cat "$TMPDIR_EVAL/register.err" 2>/dev/null)"
+fi
+
+if [[ -f "$global/projects.json" ]] && jq -e '.projects[] | select(.name == "alpha" and .profile_id == "codex-default")' "$global/projects.json" >/dev/null 2>&1; then
+  _pass "registered project preserves profile metadata"
+else
+  _fail "registered project metadata missing"
+fi
+
+echo ""
+echo "--- Sync Projects ---"
+if flow_agents_node "$USAGE_FEEDBACK" sync-projects \
+  --global-dir "$global" \
+  --repo-root "$repo_b" \
+  --name beta \
+  --profile-id codex-experimental \
+  --prompt-id verify-v1 \
+  --skill-id verify-work >"$TMPDIR_EVAL/sync.out" 2>"$TMPDIR_EVAL/sync.err"; then
+  _pass "sync-projects syncs explicit project into global root"
+else
+  _fail "sync-projects failed: $(cat "$TMPDIR_EVAL/sync.err" 2>/dev/null)"
+fi
+
+if [[ -f "$global/projects/beta/outcomes.jsonl" ]] && jq -e 'select(.repo == "beta" and .result == "failure" and .profile_id == "codex-experimental")' "$global/projects/beta/outcomes.jsonl" >/dev/null 2>&1; then
+  _pass "global project store contains synced outcome with labels"
+else
+  _fail "global project store missing synced labeled outcome"
+fi
+
+echo ""
+echo "--- Global Dashboard ---"
+if flow_agents_node "$USAGE_FEEDBACK" global-dashboard \
+  --global-dir "$global" \
+  --force >"$TMPDIR_EVAL/global-dashboard.out" 2>"$TMPDIR_EVAL/global-dashboard.err"; then
+  _pass "global-dashboard syncs registered projects and writes HTML"
+else
+  _fail "global-dashboard failed: $(cat "$TMPDIR_EVAL/global-dashboard.err" 2>/dev/null)"
+fi
+
+dashboard="$global/reports/global-dashboard.html"
+if [[ -f "$dashboard" ]] && grep -q "Usage Dashboard" "$dashboard" && grep -q "alpha" "$dashboard" && grep -q "beta" "$dashboard"; then
+  _pass "global dashboard includes multiple project groups"
+else
+  _fail "global dashboard missing expected project groups"
+fi
+
+discover_root="$TMPDIR_EVAL/discover-root"
+repo_c="$discover_root/gamma"
+mkdir -p "$repo_c/.flow-agents/gamma"
+cat > "$repo_c/.flow-agents/gamma/gamma--deliver.md" <<'MARKDOWN'
+# Deliver Gamma
+
+status: delivered
+type: deliver
+MARKDOWN
+
+if flow_agents_node "$USAGE_FEEDBACK" global-dashboard \
+  --global-dir "$global" \
+  --discover "$discover_root" \
+  --force >/dev/null 2>"$TMPDIR_EVAL/discover.err" && \
+   [[ -f "$global/projects/gamma/outcomes.jsonl" ]]; then
+  _pass "global-dashboard can discover child project directories"
+else
+  _fail "global-dashboard discovery failed: $(cat "$TMPDIR_EVAL/discover.err" 2>/dev/null)"
+fi
+
+echo ""
+echo "Result: $pass passed, $fail failed"
+[[ $fail -eq 0 ]]