@kontourai/flow-agents 1.4.0 → 2.0.0

package/evals/integration/test_resolvefirststep_security.sh

@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+# test_resolvefirststep_security.sh — Security regression for resolveFirstStep path traversal.
+#
+# Fix: resolveFirstStep in workflow-sidecar.ts previously constructed the flow-definition
+# path WITHOUT validation, allowing a crafted --flow-id like "a.../../secret" to escape
+# the kits/ directory via path.join traversal. The fix imports and reuses resolveFlowFilePath
+# from flow-resolver.ts (which already enforces SLUG_RE + path-containment), ensuring DRY
+# defense-in-depth with a single implementation.
+#
+# Tests:
+#   1. PRE-FIX proof (via resolveFlowFilePath unit): traversal inputs return null.
+#   2. POST-FIX behavioral: ensure-session --flow-id with traversal IDs
+#      produces no active_step_id (resolveFirstStep returns null → no step set).
+#   3. No out-of-tree file reads: a secret file outside kits/ is NOT read.
+#   4. Legit ensure-session --flow-id builder.build still works (first step resolved).
+#
+# Deterministic, no model spend, self-cleaning.
+# Usage: bash evals/integration/test_resolvefirststep_security.sh
+
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT/evals/lib/node.sh"
+
+TMP="$(mktemp -d)"
+errors=0
+
+_pass() { echo "  PASS: $1"; }
+_fail() { echo "  FAIL: $1"; errors=$((errors + 1)); }
+
+cleanup() { rm -rf "$TMP"; }
+trap cleanup EXIT
+
+WRITER="workflow-sidecar"
+FLOW_RESOLVER_JS="$ROOT/build/src/li""b/flow-resolver.js"
+
+echo ""
+echo "================================================================="
+echo " resolveFirstStep Path Traversal Security Regression"
+echo "================================================================="
+
+# ─── Unit: resolveFlowFilePath rejects traversal slugs ────────────────────────
+echo ""
+echo "=== 1. resolveFlowFilePath unit: traversal inputs → null (SLUG_RE defense) ==="
+
+node --input-type=module << JSEOF 2>&1
+import { resolveFlowFilePath } from '${FLOW_RESOLVER_JS}';
+
+const cases = [
+  // [kitId, flowName, flowId, repoRoot, expected]
+  ["a",        "../../secret",     "a.../../secret",     "/repo", null,  "flowName with ../ escape"],
+  ["a",        "../../../etc",     "a../../../etc",      "/repo", null,  "flowName multi-level escape"],
+  ["../evil",  "build",            "../evil.build",      "/repo", null,  "kitId with ../ escape"],
+  ["a",        "b/c",              "a.b/c",              "/repo", null,  "flowName with path separator"],
+  ["a",        "ok",               "a.ok",               "/repo", "string", "legit (a.ok) → non-null path"],
+  ["builder",  "build",            "builder.build",      "/repo", "string", "legit (builder.build) → non-null path"],
+];
+
+let failures = 0;
+for (const [kitId, flowName, flowId, repoRoot, expected, label] of cases) {
+  const result = resolveFlowFilePath(kitId, flowName, flowId, repoRoot);
+  const ok = expected === null ? result === null : (result !== null && typeof result === 'string');
+  if (!ok) {
+    console.error('  FAIL: ' + label + ' got ' + JSON.stringify(result) + ' expected ' + expected);
+    failures++;
+  } else {
+    console.log('  PASS: ' + label);
+  }
+}
+if (failures > 0) process.exit(1);
+JSEOF
+
+if [ $? -eq 0 ]; then
+  _pass "resolveFlowFilePath: all traversal inputs → null; legit inputs → valid path"
+else
+  _fail "resolveFlowFilePath: some cases did not match expected"
+fi
+
+# ─── Behavioral: ensure-session with traversal --flow-id → null active_step_id ─
+echo ""
+echo "=== 2. ensure-session --flow-id traversal → no active_step_id (null return) ==="
+
+# Create a fake "secret" file OUTSIDE kits/ to prove it is not read
+SECRET_DIR="$TMP/secret-outside"
+mkdir -p "$SECRET_DIR"
+printf 'SECRET_CONTENTS' > "$SECRET_DIR/secret.flow.json"
+
+AROOT="$TMP/traversal-aroot"
+mkdir -p "$AROOT"
+
+for traversal_id in "a.../../secret" "a.../../../etc" "builder../../../escape"; do
+  slug="trav-$(echo "$traversal_id" | tr -d './')"
+  set +e
+  flow_agents_node "$WRITER" ensure-session \
+    --artifact-root "$AROOT" \
+    --task-slug "$slug" \
+    --title "Traversal traversal test" \
+    --summary "Traversal flow-id should not escape kits/." \
+    --flow-id "$traversal_id" \
+    --timestamp "2026-06-27T00:00:00Z" >"$TMP/traversal-$slug.out" 2>&1
+  ens_exit=$?
+  set -e
+
+  # The session may succeed or fail (behavior doesn't matter); what matters is:
+  # 1. No active_step_id is set (resolveFirstStep returned null)
+  # 2. The secret file was not read (no process.env traversal occurred)
+  if [ -f "$AROOT/current.json" ]; then
+    active_step=$(node -e "
+      const fs = require('fs');
+      const c = JSON.parse(fs.readFileSync('$AROOT/current.json', 'utf8'));
+      console.log(c.active_step_id || '');
+    " 2>/dev/null || echo "")
+    if [ -z "$active_step" ]; then
+      _pass "ensure-session --flow-id '$traversal_id' → active_step_id is empty (resolveFirstStep returned null)"
+    else
+      _fail "ensure-session --flow-id '$traversal_id' → unexpected active_step_id='$active_step' (traversal may have succeeded)"
+    fi
+  else
+    # If session creation failed entirely, that's also acceptable (fail-closed)
+    _pass "ensure-session --flow-id '$traversal_id' → session not created (fail-closed)"
+  fi
+done
+
+# ─── No out-of-tree reads: FLOW_AGENTS_FLOW_DEFS_DIR .flow-agents override rejected ─
+echo ""
+echo "=== 3. FLOW_AGENTS_FLOW_DEFS_DIR pointing into .flow-agents → rejected ==="
+
+AGENT_DEFS_DIR="$TMP/agent-defs-aroot/.flow-agents/defs"
+mkdir -p "$AGENT_DEFS_DIR"
+# Write a fake flow.json in the agent-writable area
+printf '{"id":"evil.inject","steps":[{"id":"evil-step"}]}' > "$AGENT_DEFS_DIR/evil.inject.flow.json"
+
+OVERRIDE_AROOT="$TMP/override-aroot"
+mkdir -p "$OVERRIDE_AROOT"
+
+set +e
+FLOW_AGENTS_FLOW_DEFS_DIR="$AGENT_DEFS_DIR" \
+  flow_agents_node "$WRITER" ensure-session \
+    --artifact-root "$OVERRIDE_AROOT" \
+    --task-slug "evil-inject" \
+    --title "Override test" \
+    --summary "FLOW_AGENTS_FLOW_DEFS_DIR pointing into .flow-agents should be rejected." \
+    --flow-id "evil.inject" \
+    --timestamp "2026-06-27T00:00:00Z" >"$TMP/override.out" 2>&1
+set -e
+
+if [ -f "$OVERRIDE_AROOT/current.json" ]; then
+  override_step=$(node -e "
+    const fs = require('fs');
+    const c = JSON.parse(fs.readFileSync('$OVERRIDE_AROOT/current.json', 'utf8'));
+    console.log(c.active_step_id || '');
+  " 2>/dev/null || echo "")
+  if [ -z "$override_step" ]; then
+    _pass "FLOW_AGENTS_FLOW_DEFS_DIR into .flow-agents → active_step_id empty (override rejected, fell back to kits/)"
+  else
+    _fail "FLOW_AGENTS_FLOW_DEFS_DIR into .flow-agents → active_step_id='$override_step' (agent-writable override was NOT rejected)"
+  fi
+else
+  _pass "FLOW_AGENTS_FLOW_DEFS_DIR into .flow-agents → session not created (fail-closed)"
+fi
+
+# ─── Legit case: builder.build still resolves the first step ─────────────────
+echo ""
+echo "=== 4. Legit --flow-id builder.build → active_step_id set (first step resolved) ==="
+
+LEGIT_AROOT="$TMP/legit-aroot"
+mkdir -p "$LEGIT_AROOT"
+
+set +e
+flow_agents_node "$WRITER" ensure-session \
+  --artifact-root "$LEGIT_AROOT" \
+  --task-slug "legit-builder" \
+  --title "Legit builder test" \
+  --summary "builder.build should activate with a first step." \
+  --flow-id "builder.build" \
+  --timestamp "2026-06-27T00:00:00Z" >"$TMP/legit.out" 2>&1
+legit_exit=$?
+set -e
+
+legit_step=$(node -e "
+  const fs = require('fs');
+  const c = JSON.parse(fs.readFileSync('$LEGIT_AROOT/current.json', 'utf8'));
+  console.log(c.active_step_id || '');
+" 2>/dev/null || echo "")
+
+if [ -n "$legit_step" ]; then
+  _pass "ensure-session --flow-id builder.build → active_step_id='$legit_step' (first step resolved)"
+else
+  _fail "ensure-session --flow-id builder.build → active_step_id is empty (resolution failed)"
+fi
+
+# ─── Summary ─────────────────────────────────────────────────────────────────
+echo ""
+echo "================================================================="
+if [ "$errors" -eq 0 ]; then
+  echo "PASS  resolveFirstStep security eval: all checks passed."
+  echo ""
+  echo "Security fix summary:"
+  echo "  PRE-FIX:  resolveFirstStep built path directly from flowId without SLUG_RE validation."
+  echo "            A crafted --flow-id like 'a.../../secret' escaped kits/ via path.join."
+  echo "  POST-FIX: resolveFlowFilePath (from flow-resolver.ts) is reused — single implementation."
+  echo "            SLUG_RE rejects any flowName containing '../' or '/' → null returned."
+  echo "            Path-containment belt-and-suspenders confirms resolved path is inside root."
+  echo "            FLOW_AGENTS_FLOW_DEFS_DIR override pointing into .flow-agents is rejected."
+  exit 0
+fi
+echo "FAIL  resolveFirstStep security eval: $errors check(s) failed."
+exit 1

package/evals/integration/test_session_resume_roundtrip.sh

@@ -0,0 +1,286 @@
+#!/usr/bin/env bash
+# test_session_resume_roundtrip.sh — resumable-sessions (issue #153) round-trip eval
+#
+# Seeds a temporary repo fixture with an active session, runs the workflow-steering
+# hook with a SessionStart event, and asserts:
+#   AC1: RESUME block is present with status/phase/next_action/plan/handoff/trust fields
+#   AC2: Liveness warning present when a fresh other-actor event is seeded
+#   AC3: state.json / handoff.json / trust.bundle checksums are unchanged (non-destructive)
+#
+# Negative cases:
+#   - UserPromptSubmit → no RESUME block
+#   - Empty liveness stream → no LIVENESS WARNING
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+TMPDIR_EVAL="$(mktemp -d)"
+errors=0
+
+cleanup() {
+  rm -rf "$TMPDIR_EVAL"
+}
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+
+# ─── Portable sha256 helper ────────────────────────────────────────────────────
+sha256_file() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$1" | awk '{print $1}'
+  else
+    shasum -a 256 "$1" | awk '{print $1}'
+  fi
+}
+
+# ─── Seed fixture ─────────────────────────────────────────────────────────────
+REPO="$TMPDIR_EVAL/repo"
+SLUG="test-slug-153"
+TASK_DIR="$REPO/.flow-agents/$SLUG"
+mkdir -p "$TASK_DIR"
+mkdir -p "$REPO/.flow-agents/liveness"
+mkdir -p "$REPO/docs"
+
+printf '# Test Repo\n' > "$REPO/AGENTS.md"
+printf '# Context Map\n' > "$REPO/docs/context-map.md"
+
+# state.json — active session in_progress/execution
+cat > "$TASK_DIR/state.json" << 'JSON'
+{
+  "schema_version": "1.0",
+  "task_slug": "test-slug-153",
+  "status": "in_progress",
+  "phase": "execution",
+  "updated_at": "2026-06-25T00:00:00Z",
+  "next_action": {
+    "status": "active",
+    "summary": "Continue implementing the RESUME block in workflow-steering.js",
+    "target_phase": "verification"
+  },
+  "artifact_paths": ["test-slug-153--plan-work.md"]
+}
+JSON
+
+# handoff.json
+cat > "$TASK_DIR/handoff.json" << 'JSON'
+{
+  "schema_version": "1.0",
+  "task_slug": "test-slug-153",
+  "next_steps": ["Run eval and check RESUME output"],
+  "blockers": []
+}
+JSON
+
+# stub plan file
+printf '# Plan: test-slug-153\n' > "$TASK_DIR/test-slug-153--plan-work.md"
+
+# trust.bundle — one verified, one disputed
+cat > "$TASK_DIR/trust.bundle" << 'JSON'
+{
+  "schema_version": "1.0",
+  "task_slug": "test-slug-153",
+  "claims": [
+    {
+      "id": "verified-claim-001",
+      "status": "verified",
+      "claimType": "implementation",
+      "value": "feature implemented"
+    },
+    {
+      "id": "disputed-claim-id",
+      "status": "disputed",
+      "claimType": "test-coverage",
+      "value": "tests pass"
+    }
+  ]
+}
+JSON
+
+# install.json — initial version
+cat > "$REPO/.flow-agents/install.json" << 'JSON'
+{
+  "version": "v0.0.1",
+  "installedAt": "2026-06-25T00:00:00Z",
+  "runtime": "claude-code"
+}
+JSON
+
+# Liveness stream: fresh other-agent event (5 min ago, within 1800 s TTL)
+# and a self (local) event — self should NOT trigger a warning
+FIVE_MIN_AGO="$(node -e "process.stdout.write(new Date(Date.now()-300000).toISOString().replace(/\\.\\d{3}Z$/,'Z'))")"
+printf '{"type":"claim","subjectId":"test-slug-153","actor":"other-agent","at":"%s","ttlSeconds":1800}\n' "$FIVE_MIN_AGO" > "$REPO/.flow-agents/liveness/events.jsonl"
+printf '{"type":"heartbeat","subjectId":"test-slug-153","actor":"local","at":"%s"}\n' "$FIVE_MIN_AGO" >> "$REPO/.flow-agents/liveness/events.jsonl"
+
+# ─── Snapshot checksums before hook run ───────────────────────────────────────
+CKSUM_STATE_BEFORE="$(sha256_file "$TASK_DIR/state.json")"
+CKSUM_HANDOFF_BEFORE="$(sha256_file "$TASK_DIR/handoff.json")"
+CKSUM_TRUST_BEFORE="$(sha256_file "$TASK_DIR/trust.bundle")"
+
+# ─── Hot-upgrade simulation: bump install.json version ───────────────────────
+node -e "
+const fs = require('fs');
+const f = '$REPO/.flow-agents/install.json';
+const obj = JSON.parse(fs.readFileSync(f,'utf8'));
+obj.version = 'v0.0.2';
+fs.writeFileSync(f, JSON.stringify(obj, null, 2) + '\n');
+"
+
+# ─── Run hook with SessionStart ───────────────────────────────────────────────
+if echo "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"$REPO\"}" | \
+  FLOW_AGENTS_ACTOR="local" node "$ROOT/scripts/hooks/workflow-steering.js" > "$TMPDIR_EVAL/resume.out" 2>&1; then
+  _pass "hook exits 0 for SessionStart"
+else
+  _fail "hook should exit 0 for SessionStart (exit $?)"
+fi
+
+# ─── AC1: RESUME block presence and fields ────────────────────────────────────
+if grep -q "RESUME:" "$TMPDIR_EVAL/resume.out"; then
+  _pass "RESUME block present in SessionStart output"
+else
+  _fail "RESUME block missing from SessionStart output: $(cat "$TMPDIR_EVAL/resume.out")"
+fi
+
+if grep -q "in_progress" "$TMPDIR_EVAL/resume.out"; then
+  _pass "status 'in_progress' echoed in RESUME block"
+else
+  _fail "status missing from RESUME block"
+fi
+
+if grep -q "execution" "$TMPDIR_EVAL/resume.out"; then
+  _pass "phase 'execution' echoed in RESUME block"
+else
+  _fail "phase missing from RESUME block"
+fi
+
+if grep -q "Continue implementing the RESUME block" "$TMPDIR_EVAL/resume.out"; then
+  _pass "full next_action summary present in RESUME block"
+else
+  _fail "next_action summary missing from RESUME block: $(grep 'Next action' "$TMPDIR_EVAL/resume.out" || echo 'no Next action line')"
+fi
+
+if grep -q "test-slug-153--plan-work.md" "$TMPDIR_EVAL/resume.out"; then
+  _pass "plan artifact path present in RESUME block"
+else
+  _fail "plan artifact path missing from RESUME block"
+fi
+
+if grep -q "Run eval and check RESUME output" "$TMPDIR_EVAL/resume.out"; then
+  _pass "handoff next_step present in RESUME block"
+else
+  _fail "handoff next_step missing from RESUME block"
+fi
+
+if grep -q "disputed" "$TMPDIR_EVAL/resume.out"; then
+  _pass "trust takeaway mentions disputed status"
+else
+  _fail "trust takeaway missing disputed status"
+fi
+
+if grep -q "disputed-claim-id" "$TMPDIR_EVAL/resume.out"; then
+  _pass "disputed claim id present in RESUME block"
+else
+  _fail "disputed claim id missing from RESUME block"
+fi
+
+if grep -q "workflow:sidecar -- claim" "$TMPDIR_EVAL/resume.out"; then
+  _pass "disputed claim remedy command present in RESUME block"
+else
+  _fail "disputed claim remedy command missing from RESUME block"
+fi
+
+if grep -q "pull-work" "$TMPDIR_EVAL/resume.out"; then
+  _pass "pull-work route hint present in RESUME block"
+else
+  _fail "pull-work route hint missing from RESUME block"
+fi
+
+# ─── AC2: Liveness warning present ────────────────────────────────────────────
+if grep -q "LIVENESS WARNING" "$TMPDIR_EVAL/resume.out"; then
+  _pass "LIVENESS WARNING present in RESUME block"
+else
+  _fail "LIVENESS WARNING missing from RESUME block: $(cat "$TMPDIR_EVAL/resume.out")"
+fi
+
+if grep -q "other-agent" "$TMPDIR_EVAL/resume.out"; then
+  _pass "other-agent actor named in liveness warning"
+else
+  _fail "other-agent actor missing from liveness warning"
+fi
+
+# Self-actor (local) should NOT appear as a liveness warning
+if ! grep -q "LIVENESS WARNING.*local\|local.*LIVENESS WARNING" "$TMPDIR_EVAL/resume.out"; then
+  _pass "self-actor (local) correctly excluded from liveness warning"
+else
+  _fail "self-actor should not be warned in liveness advisory"
+fi
+
+# ─── AC3: Checksums unchanged (non-destructive) ───────────────────────────────
+CKSUM_STATE_AFTER="$(sha256_file "$TASK_DIR/state.json")"
+CKSUM_HANDOFF_AFTER="$(sha256_file "$TASK_DIR/handoff.json")"
+CKSUM_TRUST_AFTER="$(sha256_file "$TASK_DIR/trust.bundle")"
+
+if [[ "$CKSUM_STATE_BEFORE" == "$CKSUM_STATE_AFTER" ]]; then
+  _pass "state.json checksum unchanged (non-destructive)"
+else
+  _fail "state.json was modified by the hook (checksums differ)"
+fi
+
+if [[ "$CKSUM_HANDOFF_BEFORE" == "$CKSUM_HANDOFF_AFTER" ]]; then
+  _pass "handoff.json checksum unchanged (non-destructive)"
+else
+  _fail "handoff.json was modified by the hook (checksums differ)"
+fi
+
+if [[ "$CKSUM_TRUST_BEFORE" == "$CKSUM_TRUST_AFTER" ]]; then
+  _pass "trust.bundle checksum unchanged (non-destructive)"
+else
+  _fail "trust.bundle was modified by the hook (checksums differ)"
+fi
+
+# ─── Negative: UserPromptSubmit should produce NO RESUME block ────────────────
+echo "{\"hook_event_name\":\"UserPromptSubmit\",\"cwd\":\"$REPO\",\"prompt\":\"continue\"}" | \
+  FLOW_AGENTS_ACTOR="local" node "$ROOT/scripts/hooks/workflow-steering.js" > "$TMPDIR_EVAL/prompt.out" 2>&1
+
+if ! grep -q "RESUME:" "$TMPDIR_EVAL/prompt.out"; then
+  _pass "RESUME block absent for UserPromptSubmit (negative case)"
+else
+  _fail "RESUME block must not appear for UserPromptSubmit"
+fi
+
+# ─── Negative: Empty liveness stream → no LIVENESS WARNING ───────────────────
+REPO2="$TMPDIR_EVAL/repo2"
+TASK_DIR2="$REPO2/.flow-agents/$SLUG"
+mkdir -p "$TASK_DIR2"
+mkdir -p "$REPO2/docs"
+printf '# Test Repo 2\n' > "$REPO2/AGENTS.md"
+printf '# Context Map\n' > "$REPO2/docs/context-map.md"
+cp "$TASK_DIR/state.json" "$TASK_DIR2/state.json"
+cp "$TASK_DIR/handoff.json" "$TASK_DIR2/handoff.json"
+cp "$TASK_DIR/trust.bundle" "$TASK_DIR2/trust.bundle"
+printf 'test-slug-153--plan-work.md stub\n' > "$TASK_DIR2/test-slug-153--plan-work.md"
+# No liveness directory → empty stream
+
+echo "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"$REPO2\"}" | \
+  FLOW_AGENTS_ACTOR="local" node "$ROOT/scripts/hooks/workflow-steering.js" > "$TMPDIR_EVAL/nolive.out" 2>&1
+
+if grep -q "RESUME:" "$TMPDIR_EVAL/nolive.out"; then
+  _pass "RESUME block present when no liveness stream (absence case)"
+else
+  _fail "RESUME block should still be present with empty liveness stream"
+fi
+
+if ! grep -q "LIVENESS WARNING" "$TMPDIR_EVAL/nolive.out"; then
+  _pass "no LIVENESS WARNING when liveness stream is empty (absence case)"
+else
+  _fail "LIVENESS WARNING must not appear when no fresh other-actor events exist"
+fi
+
+# ─── Summary ──────────────────────────────────────────────────────────────────
+if [[ "$errors" -eq 0 ]]; then
+  echo "Session resume roundtrip eval passed."
+  exit 0
+fi
+
+echo "Session resume roundtrip eval failed: $errors issue(s)."
+exit 1