@kontourai/flow-agents 0.1.1

package/evals/promptfooconfig.yaml

@@ -0,0 +1,70 @@
+description: Flow Agents Eval Suite — dev skill activation and workflow compliance
+providers:
+- id: 'exec: bash lib/kiro-dev.sh'
+  label: dev
+prompts:
+- '{{prompt}}'
+tests:
+- vars:
+    prompt: Explore this codebase and tell me what it does
+  assert:
+  - type: javascript
+    value: file://lib/assertions/delegated-to.js
+    config:
+      expected:
+      - tool-explore-structure
+      - tool-explore-entry
+      - tool-explore-deps
+  - type: javascript
+    value: file://lib/assertions/tool-called.js
+    config:
+      tool: delegate to a specialist agent
+  - type: javascript
+    value: file://lib/assertions/no-write-tools.js
+  - type: llm-rubric
+    value: 'The agent activated the ''explore'' skill and fanned out parallel subagents
+
+      (tool-explore-structure, entry, deps, patterns, config, tests) in Wave 1.
+
+      Response contains a synthesis of directory structure, entry points, dependencies, and patterns.
+
+      '
+  options:
+    provider: 'exec: bash lib/kiro-dev.sh'
+- vars:
+    prompt: Create a simple hello.py script that prints hello world
+  assert:
+  - type: javascript
+    value: file://lib/assertions/tool-called.js
+    config:
+      tool: todo tool
+  - type: javascript
+    value: file://lib/assertions/no-write-tools.js
+  - type: llm-rubric
+    value: 'The dev agent followed its Phase 0-5 workflow: checked existing TODOs,
+
+      ran git status, created a plan, then attempted implementation.
+
+      Write tools are blocked so implementation may fail — check it ATTEMPTED the workflow phases.
+
+      '
+  options:
+    provider: 'exec: bash lib/kiro-dev.sh'
+- vars:
+    prompt: Check this project for outdated dependencies and security vulnerabilities
+  assert:
+  - type: javascript
+    value: file://lib/assertions/delegated-to.js
+    config:
+      expected:
+      - tool-dependencies-updater
+  - type: javascript
+    value: file://lib/assertions/no-write-tools.js
+  - type: llm-rubric
+    value: 'The agent activated the dependency-update skill and delegated to
+
+      tool-dependencies-updater rather than manually checking package files.
+
+      '
+  options:
+    provider: 'exec: bash lib/kiro-dev.sh'

package/evals/run.sh

@@ -0,0 +1,309 @@
+#!/usr/bin/env bash
+# run.sh — Entry point for the agent eval suite
+# Usage:
+#   bash run.sh              # Run layers 1+2 (fast, no LLM)
+#   bash run.sh static       # Layer 1 only
+#   bash run.sh integration  # Layer 2 only
+#   bash run.sh acceptance   # Layer 4: harness-native smoke tests
+#   bash run.sh acceptance kiro
+#   bash run.sh llm          # Layer 3: all agents
+#   bash run.sh llm dev      # Layer 3: dev agent only
+#   bash run.sh llm dev --runtime codex  # Run dev evals through Codex
+#   bash run.sh llm dev --runtime claude --judge-runtime codex
+#   bash run.sh llm dev --suite regression
+#   bash run.sh report dev   # Generate report from last run
+#   bash run.sh llm dev --repeat 3  # Run with pass@k measurement
+set -uo pipefail
+
+EVAL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$EVAL_DIR/.." && pwd)"
+LAYER="${1:-all}"
+AGENT="${2:-}"
+RUNTIME="${FLOW_AGENTS_EVAL_RUNTIME:-${EVAL_RUNTIME:-kiro}}"
+JUDGE_RUNTIME="${FLOW_AGENTS_EVAL_JUDGE_RUNTIME:-${EVAL_JUDGE_RUNTIME:-}}"
+SUITE="${FLOW_AGENTS_EVAL_SUITE:-full}"
+PROMPTFOO_BIN="${PROMPTFOO_BIN:-$ROOT_DIR/node_modules/.bin/promptfoo}"
+if [[ ! -x "$PROMPTFOO_BIN" ]]; then
+  PROMPTFOO_BIN="$(command -v promptfoo 2>/dev/null || true)"
+fi
+
+run_promptfoo() {
+  if [[ -z "$PROMPTFOO_BIN" ]]; then
+    echo "promptfoo is not installed. Run 'npm install' from the repo root." >&2
+    return 127
+  fi
+  local config_dir="${PROMPTFOO_CONFIG_DIR:-$ROOT_DIR/.promptfoo}"
+  mkdir -p "$config_dir"
+  PROMPTFOO_CONFIG_DIR="$config_dir" \
+    PROMPTFOO_DISABLE_WAL_MODE="${PROMPTFOO_DISABLE_WAL_MODE:-true}" \
+    PROMPTFOO_DISABLE_TELEMETRY="${PROMPTFOO_DISABLE_TELEMETRY:-true}" \
+    "$PROMPTFOO_BIN" "$@"
+}
+
+parse_runtime_args() {
+  local rest=()
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --runtime)
+        if [[ -z "${2:-}" ]]; then
+          echo "--runtime requires kiro or codex" >&2
+          exit 1
+        fi
+        RUNTIME="$2"
+        shift 2
+        ;;
+      --runtime=*)
+        RUNTIME="${1#--runtime=}"
+        shift
+        ;;
+      --judge-runtime)
+        if [[ -z "${2:-}" ]]; then
+          echo "--judge-runtime requires kiro, codex, or claude" >&2
+          exit 1
+        fi
+        JUDGE_RUNTIME="$2"
+        shift 2
+        ;;
+      --judge-runtime=*)
+        JUDGE_RUNTIME="${1#--judge-runtime=}"
+        shift
+        ;;
+      --suite)
+        if [[ -z "${2:-}" ]]; then
+          echo "--suite requires smoke, regression, capability, or full" >&2
+          exit 1
+        fi
+        SUITE="$2"
+        shift 2
+        ;;
+      --suite=*)
+        SUITE="${1#--suite=}"
+        shift
+        ;;
+      *)
+        rest+=("$1")
+        shift
+        ;;
+    esac
+  done
+  case "$RUNTIME" in
+    kiro|Claude\ Code|codex|claude|claude-code) ;;
+    *)
+      echo "Unsupported eval runtime '$RUNTIME' (expected kiro, codex, or claude)" >&2
+      exit 1
+      ;;
+  esac
+  JUDGE_RUNTIME="${JUDGE_RUNTIME:-$RUNTIME}"
+  case "$JUDGE_RUNTIME" in
+    kiro|Claude\ Code|codex|claude|claude-code) ;;
+    *)
+      echo "Unsupported judge runtime '$JUDGE_RUNTIME' (expected kiro, codex, or claude)" >&2
+      exit 1
+      ;;
+  esac
+  case "$SUITE" in
+    smoke)
+      rest=(--filter-first-n 3 "${rest[@]}")
+      ;;
+    regression)
+      rest=(--filter-metadata type=regression "${rest[@]}")
+      ;;
+    capability)
+      rest=(--filter-metadata type=capability "${rest[@]}")
+      ;;
+    full|"")
+      ;;
+    *)
+      echo "Unsupported suite '$SUITE' (expected smoke, regression, capability, or full)" >&2
+      exit 1
+      ;;
+  esac
+  EVAL_ARGS=("${rest[@]}")
+}
+
+run_static() {
+  echo ""
+  echo "╔══════════════════════════════════════╗"
+  echo "║  Layer 1: Static Package Validation  ║"
+  echo "╚══════════════════════════════════════╝"
+  local result=0
+  bash "$EVAL_DIR/static/test_package.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/static/test_universal_bundles.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/static/test_workflow_skills.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/static/test_evidence_refs.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/static/test_repo_hooks.sh" || result=1
+  return $result
+}
+
+run_integration() {
+  echo ""
+  echo "╔══════════════════════════════════════════╗"
+  echo "║  Layer 2: Telemetry Contract Validation  ║"
+  echo "╚══════════════════════════════════════════╝"
+  local result=0
+  bash "$EVAL_DIR/integration/test_telemetry.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_telemetry_doctor.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_usage_feedback_outcomes.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_usage_feedback_import.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_usage_feedback_report.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_usage_feedback_dashboard.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_usage_feedback_global.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_goal_fit_hook.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_hook_category_behaviors.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_workflow_artifacts.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_workflow_artifact_cleanup_audit.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_fixture_retirement_audit.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_publish_change_helper.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_workflow_sidecar_writer.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_veritas_governance_adapter.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_workflow_steering_hook.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_hook_influence_cases.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_flow_agents_statusline.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_context_map.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_flow_kit_repository.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_local_flow_kit_install.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_runtime_adapter_activation.sh" || result=1
+  echo ""
+  bash "$EVAL_DIR/integration/test_bundle_install.sh" || result=1
+  return $result
+}
+
+run_llm() {
+  parse_runtime_args "$@"
+  echo ""
+  echo "╔═══════════════════════════════════════╗"
+  echo "║  Layer 3: LLM Behavioral Evals        ║"
+  echo "╚═══════════════════════════════════════╝"
+  echo ""
+  echo "Runtime: $RUNTIME"
+  echo "Judge Runtime: ${JUDGE_RUNTIME:-$RUNTIME}"
+  echo "Suite: $SUITE"
+  echo ""
+
+  if [[ -n "$AGENT" ]]; then
+    local config="$EVAL_DIR/cases/$AGENT/promptfooconfig.yaml"
+    if [[ ! -f "$config" ]]; then
+      echo "No config found for agent '$AGENT' at $config"
+      exit 1
+    fi
+    echo "Running evals for: $AGENT"
+    cd "$EVAL_DIR/cases/$AGENT"
+    mkdir -p "$EVAL_DIR/results"
+    local output_file="$EVAL_DIR/results/${AGENT}-${RUNTIME}-$(date +%Y-%m-%d).json"
+    FLOW_AGENTS_EVAL_RUNTIME="$RUNTIME" FLOW_AGENTS_EVAL_JUDGE_RUNTIME="${JUDGE_RUNTIME:-$RUNTIME}" FLOW_AGENTS_EVAL_SUITE="$SUITE" FLOW_AGENTS_EVAL_AGENT="$AGENT" KIRO_EVAL_AGENT="$AGENT" run_promptfoo eval --no-cache --output "$output_file" "${EVAL_ARGS[@]}"
+    echo ""
+    echo "Results saved to: $output_file"
+  else
+    echo "Running all agent evals..."
+    for agent_dir in "$EVAL_DIR"/cases/*/; do
+      agent=$(basename "$agent_dir")
+      [[ ! -f "$agent_dir/promptfooconfig.yaml" ]] && continue
+      echo ""
+      echo "--- $agent ---"
+      cd "$agent_dir"
+      mkdir -p "$EVAL_DIR/results"
+      local output_file="$EVAL_DIR/results/${agent}-${RUNTIME}-$(date +%Y-%m-%d).json"
+      FLOW_AGENTS_EVAL_RUNTIME="$RUNTIME" FLOW_AGENTS_EVAL_JUDGE_RUNTIME="${JUDGE_RUNTIME:-$RUNTIME}" FLOW_AGENTS_EVAL_SUITE="$SUITE" FLOW_AGENTS_EVAL_AGENT="$agent" KIRO_EVAL_AGENT="$agent" run_promptfoo eval --no-cache --output "$output_file" "${EVAL_ARGS[@]}"
+    done
+  fi
+  echo ""
+  echo "View results: npm run promptfoo:view"
+}
+
+run_acceptance() {
+  echo ""
+  echo "╔═══════════════════════════════════════╗"
+  echo "║  Layer 4: Harness Acceptance         ║"
+  echo "╚═══════════════════════════════════════╝"
+  echo ""
+  local target="${AGENT:-all}"
+  bash "$EVAL_DIR/acceptance/run.sh" "$target"
+}
+
+run_report() {
+  local agent="${1:?Usage: bash run.sh report <agent>}"
+  local latest
+  latest=$(ls -t "$EVAL_DIR/results/${agent}"-*.json 2>/dev/null | head -1)
+  if [[ -z "$latest" ]]; then
+    echo "No results found for agent '$agent' in $EVAL_DIR/results/"
+    exit 1
+  fi
+  local previous
+  previous=$(ls -t "$EVAL_DIR/results/${agent}"-*.json 2>/dev/null | sed -n '2p')
+
+  echo ""
+  echo "╔══════════════════════════════╗"
+  echo "║  Eval Report: $agent"
+  echo "╚══════════════════════════════╝"
+  echo ""
+
+  mkdir -p "$EVAL_DIR/results/reports"
+  local report_file="$EVAL_DIR/results/reports/$(date +%Y-%m-%d)-${agent}.md"
+  bash "$EVAL_DIR/lib/eval-report.sh" "$latest" "$previous" | tee "$report_file"
+  echo ""
+  echo "Report saved to: $report_file"
+}
+
+case "$LAYER" in
+  static)      run_static ;;
+  integration) run_integration ;;
+  llm)
+    shift
+    if [[ "${1:-}" == --* ]]; then
+      AGENT=""
+    else
+      AGENT="${1:-}"
+      [[ $# -gt 0 ]] && shift
+    fi
+    run_llm "$@"
+    ;;
+  acceptance)  shift; AGENT="${1:-all}"; run_acceptance ;;
+  report)      shift; run_report "$@" ;;
+  all)
+    run_static
+    static_exit=$?
+    run_integration
+    integration_exit=$?
+    echo ""
+    echo "╔══════════════════════════╗"
+    echo "║  Summary: Layers 1 + 2  ║"
+    echo "╚══════════════════════════╝"
+    echo "  Static:      $([ $static_exit -eq 0 ] && echo PASS || echo FAIL)"
+    echo "  Integration: $([ $integration_exit -eq 0 ] && echo PASS || echo FAIL)"
+    echo ""
+    if [[ $static_exit -ne 0 || $integration_exit -ne 0 ]]; then
+      echo "Fix Layer 1/2 failures before running Layer 3."
+      exit 1
+    fi
+    echo "Layers 1+2 passed. Run 'bash run.sh acceptance [kiro|claude|codex]' for harness smoke tests or 'bash run.sh llm [dev] [--runtime kiro|codex|claude] [--judge-runtime kiro|codex|claude]' for behavioral evals."
+    ;;
+  *)
+    echo "Usage: bash run.sh [static|integration|acceptance|llm|report|all] [target]"
+    exit 1
+    ;;
+esac

package/evals/static/test_evidence_refs.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+# test_evidence_refs.sh — Structured evidence reference schema checks
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT/evals/lib/node.sh"
+
+errors=0
+pass() { echo "  ✓ $1"; }
+fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+
+echo "=== Evidence Ref Schema Checks ==="
+
+if node --input-type=module <<'NODE'
+import Ajv2020 from "ajv/dist/2020.js";
+import fs from "node:fs";
+import path from "node:path";
+
+const root = process.cwd();
+const ajv = new Ajv2020({ allErrors: true });
+const acceptanceSchema = JSON.parse(fs.readFileSync(path.join(root, "schemas/workflow-acceptance.schema.json"), "utf8"));
+const evidenceSchema = JSON.parse(fs.readFileSync(path.join(root, "schemas/workflow-evidence.schema.json"), "utf8"));
+const validateAcceptance = ajv.compile(acceptanceSchema);
+const validateEvidence = ajv.compile(evidenceSchema);
+
+const acceptance = {
+  schema_version: "1.0",
+  task_slug: "structured-evidence-ref-fixture",
+  criteria: [
+    {
+      id: "AC1",
+      description: "Behavior claim cites command and source evidence.",
+      status: "pass",
+      evidence_refs: [
+        {
+          kind: "command",
+          excerpt: "npm run eval:static --silent",
+          summary: "Static evals passed."
+        },
+        {
+          kind: "source",
+          url: "https://github.com/example/repo/blob/0123456789abcdef0123456789abcdef01234567/src/index.ts#L10-L18",
+          file: "src/index.ts",
+          line_start: 10,
+          line_end: 18,
+          excerpt: "export function implementedBehavior() { return true; }"
+        }
+      ]
+    }
+  ],
+  goal_fit: {
+    status: "pass",
+    summary: "Structured refs validate."
+  }
+};
+
+const evidence = {
+  schema_version: "1.0",
+  task_slug: "structured-evidence-ref-fixture",
+  verdict: "pass",
+  checks: [
+    {
+      id: "static-eval",
+      kind: "test",
+      status: "pass",
+      command: "npm run eval:static --silent",
+      summary: "Static evals passed.",
+      artifact_refs: [
+        {
+          kind: "source",
+          file: "evals/static/test_evidence_refs.sh",
+          line_start: 1,
+          line_end: 1,
+          excerpt: "test_evidence_refs.sh - Structured evidence reference schema checks"
+        }
+      ]
+    }
+  ],
+  external_evidence: [
+    {
+      system: "github",
+      ref: {
+        kind: "provider",
+        url: "https://github.com/example/repo/actions/runs/1",
+        summary: "Provider check run."
+      },
+      summary: "Provider evidence."
+    }
+  ]
+};
+
+const legacyAcceptance = structuredClone(acceptance);
+legacyAcceptance.criteria[0].evidence_refs = ["legacy-string-ref"];
+
+const invalidSource = structuredClone(evidence);
+delete invalidSource.checks[0].artifact_refs[0].excerpt;
+
+const emptyArtifact = structuredClone(evidence);
+emptyArtifact.checks[0].artifact_refs = [{ kind: "artifact" }];
+
+const emptyCommand = structuredClone(acceptance);
+emptyCommand.criteria[0].evidence_refs = [{ kind: "command" }];
+
+const emptyProvider = structuredClone(evidence);
+emptyProvider.external_evidence[0].ref = { kind: "provider" };
+
+if (!validateAcceptance(acceptance)) {
+  throw new Error(`structured acceptance refs should validate: ${ajv.errorsText(validateAcceptance.errors)}`);
+}
+if (!validateEvidence(evidence)) {
+  throw new Error(`structured evidence refs should validate: ${ajv.errorsText(validateEvidence.errors)}`);
+}
+if (validateAcceptance(legacyAcceptance)) {
+  throw new Error("legacy string evidence refs should fail");
+}
+if (validateEvidence(invalidSource)) {
+  throw new Error("source refs missing excerpt should fail");
+}
+if (validateEvidence(emptyArtifact)) {
+  throw new Error("artifact refs without file/url and summary/excerpt should fail");
+}
+if (validateAcceptance(emptyCommand)) {
+  throw new Error("command refs without excerpt/summary/url should fail");
+}
+if (validateEvidence(emptyProvider)) {
+  throw new Error("provider refs without url should fail");
+}
+NODE
+then
+  pass "structured refs validate and incomplete refs fail"
+else
+  fail "structured evidence ref schema check failed"
+fi
+
+if [[ $errors -eq 0 ]]; then
+  echo "=== PASS ==="
+  exit 0
+else
+  echo "=== FAIL ($errors) ==="
+  exit 1
+fi