@kontourai/flow-agents 3.5.0 → 3.7.0

package/evals/integration/test_builder_step_producers.sh

@@ -59,8 +59,9 @@ if (fs.existsSync(actorRoot)) {
   files.push(...fs.readdirSync(actorRoot).filter((name) => name.endsWith('.json')).map((name) => path.join(actorRoot, name)));
 }
 for (const file of files) {
-  const current = JSON.parse(fs.readFileSync(file, 'utf8'));
-  current.active_step_id = 'design-probe';
+      const current = JSON.parse(fs.readFileSync(file, 'utf8'));
+      current.active_flow_id = 'builder.build';
+      current.active_step_id = 'design-probe';
   fs.writeFileSync(file, `${JSON.stringify(current, null, 2)}\n`);
 }
 NODE
@@ -92,14 +93,16 @@ NODE
 # ─── Helper: bootstrap a session for produce tests ───────────────────────────
 setup_session_for_produce() {
   local aroot="$1" slug="$2" step="$3"
-  mkdir -p "$aroot"
+  mkdir -p "$aroot/$slug"
+  printf '# Pull Work\n\nSelected Work Item: local:%s\n\nFixture probe decisions are reviewable here.\n' "$slug" > "$aroot/$slug/$slug--pull-work.md"
+  printf '{"schema_version":"1.0","task_slug":"%s","decision":"hold"}\n' "$slug" > "$aroot/$slug/release.json"
+  printf '{"schema_version":"1.0","task_slug":"%s","status":"learned","records":[]}\n' "$slug" > "$aroot/$slug/learning.json"
 
   flow_agents_node "workflow-sidecar" ensure-session \
     --artifact-root "$aroot" \
     --task-slug "$slug" \
     --title "Producer test: $step" \
     --summary "Test gate-claim producer at $step." \
-    --flow-id builder.build \
     --timestamp "2026-06-26T00:00:00Z" >/dev/null 2>&1
 
   flow_agents_node "workflow-sidecar" init-plan "$aroot/$slug/$slug--deliver.md" \
@@ -115,13 +118,13 @@ setup_tamper_session() {
   mkdir -p "$t_dir"
   printf '# Repo\n' > "$t_dir/AGENTS.md"
   mkdir -p "$t_dir/.kontourai/flow-agents/$slug"
+  printf '# Pull Work\n\nSelected Work Item: local:%s\n' "$slug" > "$t_dir/.kontourai/flow-agents/$slug/$slug--pull-work.md"
 
   flow_agents_node "workflow-sidecar" ensure-session \
     --artifact-root "$t_dir/.kontourai/flow-agents" \
     --task-slug "$slug" \
     --title "Tamper test: $step" \
     --summary "Testing tamper detection." \
-    --flow-id builder.build \
     --timestamp "2026-06-26T00:00:00Z" >/dev/null 2>&1
 
   flow_agents_node "workflow-sidecar" init-plan "$t_dir/.kontourai/flow-agents/$slug/$slug--deliver.md" \
@@ -142,10 +145,19 @@ test_produce_claim() {
   local aroot="$TMP/$slug/.kontourai/flow-agents"
   setup_session_for_produce "$aroot" "$slug" "$step"
 
+  local artifact
+  case "$expectation" in
+    selected-work|pickup-probe-readiness|probe-decisions-or-accepted-gaps) artifact="$aroot/$slug/$slug--pull-work.md" ;;
+    pull-request-opened) artifact="$aroot/$slug/release.json" ;;
+    decision-evidence|learning-evidence) artifact="$aroot/$slug/learning.json" ;;
+    *) _fail "$label: fixture has no declared producer artifact"; return ;;
+  esac
+
   if flow_agents_node "workflow-sidecar" record-gate-claim "$aroot/$slug" \
     --status pass \
     --summary "Test claim: $label" \
     --expectation "$expectation" \
+    --evidence-ref-json "{\"kind\":\"artifact\",\"file\":\"$artifact\",\"summary\":\"Durable producer fixture artifact.\"}" \
     --timestamp "2026-06-26T00:02:00Z" >/dev/null 2>&1; then
     _pass "$label: record-gate-claim exits 0 at $step step"
   else
@@ -169,6 +181,10 @@ test_produce_claim() {
       console.error('expected status=verified, got', target.status);
       process.exit(1);
     }
+    if (typeof target.metadata?.expected_producer !== 'string' || typeof target.metadata?.recorded_by !== 'string') {
+      console.error('gate claim must distinguish its expected producer from the actor that recorded it');
+      process.exit(1);
+    }
   " 2>/dev/null \
     && _pass "$label: bundle contains $expected_claim_type with subjectType=$expected_subject_type, status=verified" \
     || _fail "$label: bundle missing or incorrect $expected_claim_type claim"
@@ -387,10 +403,285 @@ test_tamper_blocks \
   "builder.learn.evidence" \
   "learn" "builder.learn.evidence" "release"
 
+# ─── Public CLI happy path + route-back ─────────────────────────────────────
+echo ""
+echo "=== PUBLIC CLI: complete canonical skill/operation path ==="
+flow_agents_build_ts || _fail "public CLI fixture build failed"
+PUBLIC_ROOT="$TMP/public/.kontourai/flow-agents"
+PUBLIC_SESSION="$PUBLIC_ROOT/acme-builder-901"
+
+public_flow() {
+  CODEX_SESSION_ID=builder-public-producers node "$ROOT/build/src/cli.js" workflow "$@"
+}
+
+public_review() {
+  CODEX_SESSION_ID=builder-public-reviewer node "$ROOT/build/src/cli.js" workflow "$@"
+}
+
+assert_public_step() {
+  local step="$1" skills="$2" operations="$3" status="${4:-active}"
+  local report
+  report="$(public_flow status --session-dir "$PUBLIC_SESSION" --json 2>/dev/null)" || {
+    _fail "public status failed at $step"
+    return
+  }
+  if node - "$report" "$step" "$skills" "$operations" "$status" <<'NODE'
+const [reportText, step, skills, operations, status] = process.argv.slice(2);
+const report = JSON.parse(reportText);
+const statusMatches = status === 'active-or-blocked' ? ['active', 'blocked'].includes(report.status) : report.status === status;
+if (report.definition_id !== 'builder.build' || report.current_step !== step || !statusMatches) process.exit(1);
+if ((report.next_action?.skills || []).join(',') !== skills) process.exit(2);
+if ((report.next_action?.operations || []).join(',') !== operations) process.exit(3);
+NODE
+  then
+    _pass "public run projects $step to ${skills:-no skill}${operations:+ / $operations}"
+  else
+    _fail "public run projection mismatch at $step: $report"
+  fi
+}
+
+record_public_expectation() {
+  local expectation="$1" status="${2:-pass}"
+  local artifact slug
+  slug="$(basename "$PUBLIC_SESSION")"
+  case "$expectation" in
+    shaped-problem|shaped-outcome|shaped-constraints|shaped-non-goals|shaped-success|shaped-risk|open-decisions|slices-defined|work-items-filed) artifact="$PUBLIC_SESSION/$slug--idea-to-backlog.md" ;;
+    pickup-probe-readiness|probe-decisions-or-accepted-gaps) artifact="$PUBLIC_SESSION/$slug--pull-work.md" ;;
+    implementation-plan) artifact="$PUBLIC_SESSION/$slug--plan-work.md" ;;
+    implementation-scope) artifact="$PUBLIC_SESSION/$slug--deliver.md" ;;
+    tests-evidence) artifact="$PUBLIC_SESSION/$slug--plan-work.md" ;;
+    merge-readiness) artifact="$PUBLIC_SESSION/$slug--evidence-gate.md" ;;
+    pull-request-opened|ci-merge-readiness) artifact="$PUBLIC_SESSION/release.json" ;;
+    decision-evidence|learning-evidence) artifact="$PUBLIC_SESSION/learning.json" ;;
+    *) _fail "public producer fixture has no durable artifact for $expectation"; return ;;
+  esac
+  local -a args=(evidence --session-dir "$PUBLIC_SESSION" --expectation "$expectation" --status "$status" --summary "public producer fixture records a reviewable durable artifact" --evidence-ref-json "{\"kind\":\"artifact\",\"file\":\"$artifact\",\"summary\":\"Fixture artifact for $expectation.\"}")
+  if [ "$expectation" = "tests-evidence" ] && [ "$status" = "pass" ]; then
+    local test_command criterion_one criterion_two command_ref
+    test_command="bash checks/check-public-session.sh .kontourai/flow-agents/$slug/state.json"
+    criterion_one="$(node - "$test_command" <<'NODE'
+const command = process.argv[2];
+process.stdout.write(JSON.stringify({ id: 'AC-1', status: 'pass', evidence_refs: [{ kind: 'command', excerpt: command, summary: 'Substantive fixture assertion for AC-1.' }] }));
+NODE
+)"
+    criterion_two="$(node - "$test_command" <<'NODE'
+const command = process.argv[2];
+process.stdout.write(JSON.stringify({ id: 'AC-2', status: 'pass', evidence_refs: [{ kind: 'command', excerpt: command, summary: 'Substantive fixture assertion for AC-2.' }] }));
+NODE
+)"
+    command_ref="$(node - "$test_command" <<'NODE'
+const command = process.argv[2];
+process.stdout.write(JSON.stringify({ kind: 'command', excerpt: command, summary: 'Exact project-local verification command.' }));
+NODE
+)"
+    args+=(--command "$test_command" --evidence-ref-json "$command_ref" --criterion-json "$criterion_one" --criterion-json "$criterion_two")
+  fi
+  local output
+  if ! output="$(public_flow "${args[@]}" 2>&1)"; then
+    _fail "public evidence failed for $expectation: $output"
+  fi
+}
+
+prepare_public_artifacts() {
+node - "$PUBLIC_SESSION" <<'NODE'
+const fs = require('node:fs');
+const path = require('node:path');
+const session = process.argv[2];
+const slug = path.basename(session);
+const write = (name, body) => fs.writeFileSync(path.join(session, name), body, 'utf8');
+const projectRoot = path.dirname(path.dirname(path.dirname(session)));
+const checksDir = path.join(projectRoot, 'checks');
+fs.mkdirSync(checksDir, { recursive: true });
+const checkScript = path.join(checksDir, 'check-public-session.sh');
+fs.writeFileSync(checkScript, '#!/usr/bin/env bash\nset -eu\ntest -f "$1"\nprintf "1..1\\nok 1 - session exists\\n"\n', 'utf8');
+fs.chmodSync(checkScript, 0o755);
+write(`${slug}--idea-to-backlog.md`, '# Idea To Backlog Report\n\nShaped problem, slices, and filed work are reviewable here.\n');
+write(`${slug}--pull-work.md`, '# Pull and Probe Report\n\nSelected work, scope, decisions, and accepted gaps are reviewable here.\n');
+write(`${slug}--plan-work.md`, '# Plan\n\n## Definition Of Done\n\n- AC-1: The producer fixture records criterion-backed evidence.\n- AC-2: Every accepted criterion is backed by the exact test command.\n');
+write(`${slug}--evidence-gate.md`, '# Evidence Gate\n\nAcceptance coverage and scope-integrity decision.\n');
+write('acceptance.json', JSON.stringify({ schema_version: '1.0', task_slug: slug, criteria: [{ id: 'AC-1', description: 'The producer fixture records criterion-backed evidence.', status: 'pending', evidence_refs: [] }, { id: 'AC-2', description: 'Every accepted criterion is backed by the exact test command.', status: 'pending', evidence_refs: [] }], goal_fit: { status: 'pending', summary: 'Fixture has not completed Goal Fit review.' } }, null, 2));
+write('handoff.json', JSON.stringify({ schema_version: '1.0', task_slug: slug, summary: 'Fixture execution handoff.', next_steps: ['Execute the reviewed plan.'], blockers: [] }, null, 2));
+write('release.json', JSON.stringify({ schema_version: '1.0', task_slug: slug, decision: 'hold', updated_at: '2026-06-26T00:00:00Z', scope: 'producer fixture', evidence_ref: `${slug}--evidence-gate.md`, gates: [{ name: 'merge', status: 'hold', summary: 'Fixture is not authorized to merge.' }], rollback_plan: { status: 'not_required', summary: 'No release.', owner: 'fixture' }, observability_plan: { status: 'not_required', summary: 'No release.' }, post_deploy_checks: [], docs: { status: 'not_needed', summary: 'Fixture.' } }, null, 2));
+write('learning.json', JSON.stringify({ schema_version: '1.0', task_slug: slug, status: 'learned', updated_at: '2026-06-26T00:00:00Z', records: [{ id: 'fixture-learning', recorded_at: '2026-06-26T00:00:00Z', source_refs: [`${slug}--evidence-gate.md`], outcome: 'success', facts: ['Fixture created reviewable durable artifacts.'], interpretation: 'No workflow correction is needed.', routing: [{ target: 'none', action: 'No follow-up required.', status: 'completed' }], correction: { needed: false, evidence: 'Fixture contract passes.' } }] }, null, 2));
+NODE
+}
+
+mkdir -p "$PUBLIC_ROOT"
+mkdir -p "$PUBLIC_SESSION"
+printf 'Selected Work Item: acme/builder#901\n' > "$PUBLIC_SESSION/acme-builder-901--pull-work.md"
+public_flow start --artifact-root "$PUBLIC_ROOT" --flow builder.build \
+  --work-item acme/builder#901 --assignment-provider local-file --summary "Public producer path" >/dev/null 2>&1 \
+  || _fail "public workflow start failed"
+prepare_public_artifacts
+assert_public_step "design-probe" "pickup-probe" ""
+record_public_expectation "pickup-probe-readiness"
+assert_public_step "design-probe" "pickup-probe" "" "active-or-blocked"
+record_public_expectation "probe-decisions-or-accepted-gaps"
+assert_public_step "plan" "plan-work" ""
+record_public_expectation "implementation-plan"
+assert_public_step "execute" "execute-plan" ""
+record_public_expectation "implementation-scope"
+assert_public_step "verify" "review-work,verify-work" ""
+if public_review critique --session-dir "$PUBLIC_SESSION" --verdict pass --summary "Missing lanes must fail." --artifact-ref "$PUBLIC_SESSION/$(basename "$PUBLIC_SESSION")--deliver.md" >/dev/null 2>&1; then
+  _fail "public critique accepted a passing review without lanes"
+else
+  _pass "public critique rejects a passing review without lanes"
+fi
+PUBLIC_CRITIQUE_OUTPUT="$(public_review critique --session-dir "$PUBLIC_SESSION" \
+  --verdict pass \
+  --summary "Authenticated review found no blocking fixture findings." \
+  --artifact-ref "$PUBLIC_SESSION/$(basename "$PUBLIC_SESSION")--deliver.md" \
+  --lane-json "{\"id\":\"code-review\",\"status\":\"pass\",\"summary\":\"Public fixture code review completed.\",\"evidence_refs\":[{\"kind\":\"artifact\",\"file\":\"$PUBLIC_SESSION/$(basename "$PUBLIC_SESSION")--deliver.md\",\"summary\":\"Reviewed public fixture delivery artifact.\"}]}" 2>&1)" \
+  || _fail "public authenticated critique failed before tests-evidence: $PUBLIC_CRITIQUE_OUTPUT"
+PUBLIC_TEST_COMMAND="bash checks/check-public-session.sh .kontourai/flow-agents/$(basename "$PUBLIC_SESSION")/state.json"
+if public_flow evidence --session-dir "$PUBLIC_SESSION" --expectation tests-evidence --status pass --command "bash -c true" --summary "Wrapped no-op must not count as tests evidence." \
+  --criterion-json '{"id":"AC-1","status":"pass","evidence_refs":[{"kind":"command","excerpt":"bash -c true","summary":"Wrapped no-op."}]}' \
+  --criterion-json '{"id":"AC-2","status":"pass","evidence_refs":[{"kind":"command","excerpt":"bash -c true","summary":"Wrapped no-op."}]}' >/dev/null 2>&1; then
+  _fail "public tests-evidence accepted a wrapped no-op command"
+else
+  _pass "public tests-evidence rejects a wrapped no-op command"
+fi
+if public_flow evidence --session-dir "$PUBLIC_SESSION" --expectation tests-evidence --status pass --command "$PUBLIC_TEST_COMMAND" --summary "External-only criterion evidence must not pass." \
+  --criterion-json '{"id":"AC-1","status":"pass","evidence_refs":[{"kind":"external","url":"https://example.invalid/ac-1","summary":"External attestation only."}]}' \
+  --criterion-json '{"id":"AC-2","status":"pass","evidence_refs":[{"kind":"external","url":"https://example.invalid/ac-2","summary":"External attestation only."}]}' >/dev/null 2>&1; then
+  _fail "public tests-evidence accepted external-only passing criterion evidence"
+else
+  _pass "public tests-evidence rejects external-only passing criterion evidence"
+fi
+if public_flow evidence --session-dir "$PUBLIC_SESSION" --expectation tests-evidence --status pass --command "$PUBLIC_TEST_COMMAND" --summary "Every criterion must cite the exact command." \
+  --criterion-json "{\"id\":\"AC-1\",\"status\":\"pass\",\"evidence_refs\":[{\"kind\":\"command\",\"excerpt\":\"$PUBLIC_TEST_COMMAND\",\"summary\":\"Exact command for AC-1.\"}]}" \
+  --criterion-json '{"id":"AC-2","status":"pass","evidence_refs":[{"kind":"command","excerpt":"bash checks/check-public-session.sh .kontourai/flow-agents/missing/state.json","summary":"Different command for AC-2."}]}' >/dev/null 2>&1; then
+  _fail "public tests-evidence accepted a criterion without the exact command"
+else
+  _pass "public tests-evidence requires the exact command for every criterion"
+fi
+record_public_expectation "tests-evidence"
+if node - "$PUBLIC_SESSION/trust.bundle" <<'NODE'
+const fs = require('node:fs');
+const bundle = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
+const verified = (claimType, subjectType) => bundle.claims.some((claim) => (
+  claim.claimType === claimType
+  && claim.subjectType === subjectType
+  && claim.status === 'verified'
+));
+if (!verified('workflow.critique.review', 'workflow-critique')) process.exit(1);
+if (!verified('workflow.acceptance.criterion', 'flow-step')) process.exit(2);
+if (!verified('builder.verify.tests', 'flow-step')) process.exit(3);
+NODE
+then
+  _pass "public verify gate is backed by current critique, criterion, and test claims"
+else
+  _fail "public verify gate is missing one or more declared trust-bundle claims"
+fi
+assert_public_step "merge-ready" "evidence-gate" ""
+record_public_expectation "merge-readiness"
+assert_public_step "pr-open" "" "publish-change"
+record_public_expectation "pull-request-opened"
+assert_public_step "merge-ready-ci" "release-readiness" ""
+record_public_expectation "ci-merge-readiness"
+assert_public_step "learn" "learning-review" ""
+record_public_expectation "decision-evidence"
+assert_public_step "learn" "learning-review" "" "active-or-blocked"
+record_public_expectation "learning-evidence"
+if node - "$PUBLIC_SESSION/trust.bundle" <<'NODE'
+const fs = require('node:fs');
+const bundle = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
+if (!bundle.claims.some((claim) => claim.claimType === 'builder.learn.evidence' && claim.subjectType === 'release' && claim.status === 'verified')) process.exit(1);
+NODE
+then
+  _pass "learning-review records verified learning.json evidence"
+else
+  _fail "learning-review did not record durable learning.json evidence in trust.bundle"
+fi
+assert_public_step "learn" "" "" "completed"
+
+echo ""
+echo "=== PUBLIC CLI: failed verify route-back remains canonical ==="
+ROUTE_SESSION="$PUBLIC_ROOT/acme-builder-902"
+PUBLIC_SESSION="$ROUTE_SESSION"
+mkdir -p "$ROUTE_SESSION"
+printf 'Selected Work Item: acme/builder#902\n' > "$ROUTE_SESSION/acme-builder-902--pull-work.md"
+public_flow start --artifact-root "$PUBLIC_ROOT" --flow builder.build \
+  --work-item acme/builder#902 --assignment-provider local-file --summary "Public route-back path" >/dev/null 2>&1 \
+  || _fail "route-back workflow start failed"
+prepare_public_artifacts
+for expectation in pickup-probe-readiness probe-decisions-or-accepted-gaps implementation-plan implementation-scope; do
+  record_public_expectation "$expectation"
+done
+ROUTE_CRITIQUE_OUTPUT="$(public_review critique --session-dir "$PUBLIC_SESSION" \
+  --verdict pass \
+  --summary "Authenticated review completed before failed verification evidence." \
+  --artifact-ref "$PUBLIC_SESSION/$(basename "$PUBLIC_SESSION")--deliver.md" \
+  --lane-json "{\"id\":\"code-review\",\"status\":\"pass\",\"summary\":\"Public fixture code review completed.\",\"evidence_refs\":[{\"kind\":\"artifact\",\"file\":\"$PUBLIC_SESSION/$(basename "$PUBLIC_SESSION")--deliver.md\",\"summary\":\"Reviewed public fixture delivery artifact.\"}]}" 2>&1)" \
+  || _fail "public authenticated critique failed before failed tests-evidence: $ROUTE_CRITIQUE_OUTPUT"
+record_public_expectation "tests-evidence" "fail"
+ROUTE_REPORT="$(public_flow status --session-dir "$PUBLIC_SESSION" --json 2>/dev/null)"
+if node - "$ROUTE_REPORT" <<'NODE'
+const report = JSON.parse(process.argv[2]);
+if (report.current_step !== 'verify') process.exit(1);
+if ((report.next_action?.skills || []).join(',') !== 'review-work,verify-work') process.exit(2);
+if (!/attempt 1\/3 returned to `verify`/.test(report.next_action?.summary || '')) process.exit(3);
+NODE
+then
+  _pass "failed verify evidence routes back through Flow with attempt 1/3 and canonical producers"
+else
+  _fail "public failed-verify route-back was not projected correctly: $ROUTE_REPORT"
+fi
+
+echo ""
+echo "=== PUBLIC CLI: complete builder.shape path when supported ==="
+SHAPE_SESSION="$PUBLIC_ROOT/builder-shape-903"
+PUBLIC_SESSION="$SHAPE_SESSION"
+SHAPE_START_OUTPUT="$(public_flow start --artifact-root "$PUBLIC_ROOT" --flow builder.shape \
+  --task-slug builder-shape-903 --summary "Shape public producer path" 2>&1)"
+if [ "$?" -ne 0 ]; then
+  if printf '%s' "$SHAPE_START_OUTPUT" | rg -q 'supports only|unsupported.*builder\.shape|Unknown workflow'; then
+    _pass "public builder.shape path is deferred until the installed runtime exposes it"
+  else
+    _fail "public builder.shape start failed unexpectedly: $SHAPE_START_OUTPUT"
+  fi
+else
+  assert_public_shape_step() {
+    local step="$1" skills="$2" status="${3:-active}"
+    local report
+    report="$(public_flow status --session-dir "$SHAPE_SESSION" --json 2>/dev/null)" || {
+      _fail "public shape status failed at $step"
+      return
+    }
+    if node - "$report" "$step" "$skills" "$status" <<'NODE'
+const [reportText, step, skills, status] = process.argv.slice(2);
+const report = JSON.parse(reportText);
+const statusMatches = status === 'active-or-completed'
+  ? ['active', 'completed'].includes(report.status)
+  : status === 'active-or-blocked'
+    ? ['active', 'blocked'].includes(report.status)
+    : report.status === status;
+if (report.definition_id !== 'builder.shape' || report.current_step !== step || !statusMatches) process.exit(1);
+if ((report.next_action?.skills || []).join(',') !== skills) process.exit(2);
+NODE
+    then
+      _pass "public shape run projects $step to ${skills:-no skill}"
+    else
+      _fail "public shape projection mismatch at $step: $report"
+    fi
+  }
+
+  prepare_public_artifacts
+  assert_public_shape_step "shape" "idea-to-backlog" "active-or-blocked"
+  for expectation in shaped-problem shaped-outcome shaped-constraints shaped-non-goals shaped-success shaped-risk; do
+    record_public_expectation "$expectation"
+  done
+  assert_public_shape_step "breakdown" "idea-to-backlog"
+  record_public_expectation "slices-defined"
+  assert_public_shape_step "file-issues" "idea-to-backlog"
+  record_public_expectation "work-items-filed"
+  assert_public_shape_step "shape-done" "" "active-or-completed"
+fi
+
 # ─── Summary ──────────────────────────────────────────────────────────────────
 echo ""
 if [ "$errors" -eq 0 ]; then
-  echo "Builder step producer tests passed (6 claims: produce + tamper-block each)."
+  echo "Builder step producer tests passed (legacy tamper checks plus full public happy path and route-back)."
   exit 0
 fi
 echo "Builder step producer tests FAILED: $errors issue(s)."