@kontourai/flow-agents 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +12 -0
- package/CHANGELOG.md +32 -0
- package/CONTEXT.md +67 -1
- package/README.md +4 -0
- package/build/src/cli/assignment-provider.d.ts +188 -0
- package/build/src/cli/assignment-provider.js +193 -51
- package/build/src/cli/workflow-sidecar.d.ts +16 -6
- package/build/src/cli/workflow-sidecar.js +492 -45
- package/build/src/lib/flow-resolver.d.ts +12 -6
- package/build/src/lib/flow-resolver.js +30 -14
- package/build/src/tools/validate-source-tree.js +2 -1
- package/context/contracts/assignment-provider-contract.md +10 -1
- package/context/contracts/execution-contract.md +78 -0
- package/context/contracts/probe-docs-write-contract.md +187 -0
- package/context/scripts/hooks/config-protection.js +24 -4
- package/context/scripts/hooks/lib/config-protection-remedies.js +9 -0
- package/context/scripts/hooks/stop-goal-fit.js +262 -5
- package/context/scripts/hooks/workflow-steering.js +42 -0
- package/docs/adr/0001-flow-agents-consumes-flow.md +2 -0
- package/docs/adr/0002-flow-kits-as-extension-unit.md +2 -0
- package/docs/adr/0003-flow-agents-coordinates-kits-and-adapters.md +2 -0
- package/docs/adr/0004-gates-expect-surface-claims.md +2 -0
- package/docs/adr/0005-kubernetes-inspired-resource-contracts.md +2 -0
- package/docs/adr/0006-typescript-first-source-policy.md +2 -0
- package/docs/adr/0007-flow-skill-kit-tool-boundary.md +2 -0
- package/docs/adr/0007-skill-audit.md +2 -0
- package/docs/adr/0008-kit-operation-boundary.md +2 -0
- package/docs/adr/0009-canonical-hook-core-kit-boundary.md +2 -0
- package/docs/adr/0010-workflow-trust-state-as-hachure-bundle.md +2 -0
- package/docs/adr/0011-mcp-posture.md +2 -0
- package/docs/adr/0012-agent-coordination-as-liveness-claims.md +2 -0
- package/docs/adr/0013-context-lifecycle.md +2 -0
- package/docs/adr/0014-core-vs-domain-kit-boundary.md +2 -0
- package/docs/adr/0015-flow-flow-agents-boundary-reconciliation.md +2 -0
- package/docs/adr/0016-three-hard-boundary-model.md +2 -0
- package/docs/adr/0017-anti-gaming-trust-security-model.md +2 -0
- package/docs/adr/0018-freeze-local-shell-heuristics.md +2 -0
- package/docs/adr/0019-kit-dependency-ownership.md +2 -0
- package/docs/adr/0020-trust-reconcile-manifest-and-claim-classification.md +2 -0
- package/docs/adr/0021-assignment-leases-and-stale-claim-takeover.md +3 -1
- package/docs/adr/0022-fail-closed-delivery-reconciliation-with-governed-exemptions.md +113 -0
- package/docs/adr/README.md +49 -0
- package/docs/adr/index.md +34 -0
- package/docs/decisions/agent-coordination.md +20 -0
- package/docs/decisions/anti-gaming-trust-security.md +20 -0
- package/docs/decisions/context-lifecycle.md +18 -0
- package/docs/decisions/core-domain-kit-boundary.md +18 -0
- package/docs/decisions/flow-flow-agents-boundary.md +18 -0
- package/docs/decisions/flow-kit.md +20 -0
- package/docs/decisions/flow-skill-kit-tool-boundary.md +20 -0
- package/docs/decisions/graph-knowledge-provider.md +63 -0
- package/docs/decisions/hook-core-kit-boundary.md +18 -0
- package/docs/decisions/index.md +19 -0
- package/docs/decisions/kit-dependency-ownership.md +18 -0
- package/docs/decisions/kit-operation-boundary.md +18 -0
- package/docs/decisions/knowledge-promote-sub-flow.md +59 -0
- package/docs/decisions/kontour-resource-contract.md +18 -0
- package/docs/decisions/mcp-posture.md +18 -0
- package/docs/decisions/three-hard-boundary-model.md +18 -0
- package/docs/decisions/trust-reconcile.md +20 -0
- package/docs/decisions/typescript-source-policy.md +48 -0
- package/docs/decisions/workflow-enforcement.md +18 -0
- package/docs/decisions/workflow-trust-state.md +20 -0
- package/docs/fixture-ownership.md +1 -1
- package/docs/workflow-usage-guide.md +1 -1
- package/evals/ci/run-baseline.sh +6 -0
- package/evals/fixtures/assignment-provider/hostile-effective-state.json +11 -0
- package/evals/integration/test_checkpoint_signing.sh +4 -3
- package/evals/integration/test_current_json_per_actor.sh +516 -0
- package/evals/integration/test_ensure_session_ownership_guard.sh +435 -0
- package/evals/integration/test_gate_lockdown.sh +46 -0
- package/evals/integration/test_model_routing_escalation.sh +145 -0
- package/evals/integration/test_publish_delivery.sh +14 -6
- package/evals/integration/test_pull_work_liveness_preflight.sh +6 -1
- package/evals/integration/test_stop_hook_release.sh +552 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +170 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +12 -3
- package/evals/run.sh +10 -0
- package/evals/static/test_knowledge_providers.sh +13 -4
- package/evals/static/test_model_routing_hints.sh +107 -0
- package/evals/static/test_workflow_skills.sh +15 -2
- package/kits/builder/skills/builder-shape/SKILL.md +10 -0
- package/kits/builder/skills/deliver/SKILL.md +69 -11
- package/kits/builder/skills/design-probe/SKILL.md +47 -0
- package/kits/builder/skills/execute-plan/SKILL.md +13 -0
- package/kits/builder/skills/fix-bug/SKILL.md +17 -0
- package/kits/builder/skills/idea-to-backlog/SKILL.md +10 -0
- package/kits/builder/skills/pickup-probe/SKILL.md +4 -4
- package/kits/builder/skills/plan-work/SKILL.md +9 -0
- package/kits/builder/skills/pull-work/SKILL.md +10 -0
- package/kits/builder/skills/review-work/SKILL.md +11 -0
- package/kits/builder/skills/tdd-workflow/SKILL.md +17 -0
- package/kits/builder/skills/verify-work/SKILL.md +11 -0
- package/kits/knowledge/adapters/default-store/index.js +147 -18
- package/kits/knowledge/adapters/flow-runner/index.js +912 -16
- package/kits/knowledge/adapters/obsidian-store/index.js +97 -14
- package/kits/knowledge/adapters/shared/codec.js +265 -0
- package/kits/knowledge/docs/README.md +121 -2
- package/kits/knowledge/docs/store-contract.md +517 -7
- package/kits/knowledge/evals/audit-freshness/suite.test.js +92 -1
- package/kits/knowledge/evals/consolidate-incremental/suite.test.js +494 -0
- package/kits/knowledge/evals/consolidation/suite.test.js +1 -1
- package/kits/knowledge/evals/contract-suite/suite.test.js +212 -0
- package/kits/knowledge/evals/freshness/suite.test.js +339 -0
- package/kits/knowledge/evals/inbound-references/suite.test.js +351 -0
- package/kits/knowledge/evals/retirement/suite.test.js +1 -1
- package/kits/knowledge/evals/supersede-propagation/suite.test.js +384 -0
- package/kits/knowledge/flows/promote.flow.json +84 -0
- package/kits/knowledge/kit.json +15 -0
- package/kits/knowledge/promote/distill.js +96 -0
- package/kits/knowledge/promote/fixtures/contradiction-registry/CONTEXT.md +7 -0
- package/kits/knowledge/promote/fixtures/contradiction-registry/docs/decisions/cache-eviction-policy.md +12 -0
- package/kits/knowledge/promote/fixtures/contradiction-registry/docs/decisions/cache-eviction.md +12 -0
- package/kits/knowledge/promote/fixtures/repo/CONTEXT.md +11 -0
- package/kits/knowledge/promote/fixtures/repo/docs/decisions/index.md +9 -0
- package/kits/knowledge/promote/fixtures/repo/docs/decisions/liveness-claim.md +12 -0
- package/kits/knowledge/promote/fixtures/session/acceptance.json +9 -0
- package/kits/knowledge/promote/fixtures/session/agents/tool-planner/events.jsonl +1 -0
- package/kits/knowledge/promote/fixtures/session/agents/tool-verifier/events.jsonl +1 -0
- package/kits/knowledge/promote/fixtures/session/agents/tool-worker-1-1/events.jsonl +1 -0
- package/kits/knowledge/promote/fixtures/session/kontourai-flow-agents-287--deliver.md +47 -0
- package/kits/knowledge/promote/fixtures/session/learning.json +35 -0
- package/kits/knowledge/promote/fixtures/session/state.json +19 -0
- package/kits/knowledge/promote/health.js +137 -0
- package/kits/knowledge/promote/index.js +176 -0
- package/kits/knowledge/promote/ingest.js +103 -0
- package/kits/knowledge/promote/lib.js +132 -0
- package/kits/knowledge/promote/link.js +84 -0
- package/kits/knowledge/promote/promote.test.js +174 -0
- package/kits/knowledge/providers/conformance/suite.test.js +18 -0
- package/kits/knowledge/providers/index.js +1 -0
- package/kits/knowledge/providers/neo4j/connection.js +121 -0
- package/kits/knowledge/providers/neo4j/cypher.js +190 -0
- package/kits/knowledge/providers/neo4j/fake-driver.js +123 -0
- package/kits/knowledge/providers/neo4j/fixtures/spike-ground-truth.json +29 -0
- package/kits/knowledge/providers/neo4j/graph-queries.js +210 -0
- package/kits/knowledge/providers/neo4j/index.js +280 -0
- package/kits/knowledge/providers/neo4j/integration.test.js +122 -0
- package/kits/knowledge/providers/neo4j/neo4j.test.js +219 -0
- package/kits/knowledge/providers/neo4j/sync.js +235 -0
- package/package.json +4 -2
- package/schemas/workflow-handoff.schema.json +6 -0
- package/scripts/README.md +1 -0
- package/scripts/ci/mint-attestation.js +33 -6
- package/scripts/ci/trust-reconcile.js +144 -26
- package/scripts/freeze-adrs.mjs +364 -0
- package/scripts/hooks/config-protection.js +24 -4
- package/scripts/hooks/evidence-capture.js +4 -1
- package/scripts/hooks/lib/config-protection-remedies.js +9 -0
- package/scripts/hooks/lib/current-pointer.js +123 -0
- package/scripts/hooks/lib/liveness-heartbeat.js +75 -35
- package/scripts/hooks/stop-goal-fit.js +262 -5
- package/scripts/hooks/workflow-steering.js +42 -0
- package/scripts/statusline/flow-agents-statusline.js +3 -1
- package/src/cli/assignment-provider.ts +239 -59
- package/src/cli/workflow-sidecar.ts +529 -43
- package/src/lib/flow-resolver.ts +35 -14
- package/src/tools/validate-source-tree.ts +2 -1
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_ensure_session_ownership_guard.sh — ensure-session's pre-entry ownership guard (#291).
|
|
3
|
+
#
|
|
4
|
+
# Exercises `enforceEnsureSessionOwnership()` (src/cli/workflow-sidecar.ts) against a temp
|
|
5
|
+
# ARTIFACT_ROOT, reusing #290's local-file assignment ⋈ liveness join
|
|
6
|
+
# (`computeEffectiveState`/`performLocalClaim`/`performLocalSupersede`, exported in
|
|
7
|
+
# src/cli/assignment-provider.ts). Fully deterministic: two fixed test-actor identities
|
|
8
|
+
# (--actor eval-actor-a-session / eval-actor-b-session), no network, no `gh` process. Follows
|
|
9
|
+
# test_assignment_provider_local_file.sh's pass/fail/json_query idiom exactly.
|
|
10
|
+
#
|
|
11
|
+
# Plan sections (see .kontourai/flow-agents/kontourai-flow-agents-291/
|
|
12
|
+
# kontourai-flow-agents-291--plan-work.md, Wave 3 "Ownership-guard eval"):
|
|
13
|
+
# 1. Actor A ensure-session on a free subject → exit 0; a durable local-file claim is
|
|
14
|
+
# established (AC5).
|
|
15
|
+
# 2. Actor B ensure-session on the SAME subject while A's claim is fresh (a real liveness event
|
|
16
|
+
# is manufactured so the join classifies it `held`, not merely `reclaimable`) → exit nonzero,
|
|
17
|
+
# naming A's holder identity (AC1); plus two hostile-string sanitization variants (AC9): a
|
|
18
|
+
# crafted hostile LIVENESS event (liveness-only holder, no assignment record) and the
|
|
19
|
+
# `hostile-effective-state.json` fixture via --effective-state-json (github-style hostile
|
|
20
|
+
# assignee string).
|
|
21
|
+
# 3. Actor A re-running ensure-session on its OWN subject → exit 0, no refusal (AC4).
|
|
22
|
+
# 4. A manufactured stale claim (real assignment record + a liveness event whose `at` is well
|
|
23
|
+
# past its ttlSeconds) → refuses without --supersede-stale (naming the flag), succeeds with
|
|
24
|
+
# --supersede-stale (AC2, both halves; audit-trail supersede entry confirmed).
|
|
25
|
+
# 5. A human-held fixture (assignment record with actor.human set) → any actor's ensure-session
|
|
26
|
+
# → exit nonzero, ask-first remediation, no auto-reclaim (AC3).
|
|
27
|
+
# 6. Concurrency: two REAL, concurrently-launched ensure-session processes on the same fresh
|
|
28
|
+
# subject (background `&` + `wait`, genuine OS-process concurrency, not a sequential
|
|
29
|
+
# simulation) → exactly one becomes the confirmed holder; the assignment record stays valid,
|
|
30
|
+
# single-holder, single-audit-entry JSON (no corrupted/partial write, no double claim) (AC6).
|
|
31
|
+
set -uo pipefail
|
|
32
|
+
|
|
33
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
34
|
+
source "$ROOT/evals/lib/node.sh"
|
|
35
|
+
|
|
36
|
+
CLI="$ROOT/build/src/cli.js"
|
|
37
|
+
FIXTURES="$ROOT/evals/fixtures/assignment-provider"
|
|
38
|
+
HOSTILE_EFFECTIVE_STATE_FIXTURE="$FIXTURES/hostile-effective-state.json"
|
|
39
|
+
ACTOR_IDENTITY_HELPER="$ROOT/scripts/hooks/lib/actor-identity.js"
|
|
40
|
+
|
|
41
|
+
TMPDIR_EVAL="$(mktemp -d)"
|
|
42
|
+
trap 'rm -rf "$TMPDIR_EVAL"' EXIT
|
|
43
|
+
ARTIFACT_ROOT="$TMPDIR_EVAL/artifact-root"
|
|
44
|
+
|
|
45
|
+
errors=0
|
|
46
|
+
pass() { echo " ✓ $1"; }
|
|
47
|
+
fail() { echo " ✗ $1"; errors=$((errors + 1)); }
|
|
48
|
+
|
|
49
|
+
json_query() {
|
|
50
|
+
node -e 'const fs=require("fs"); let cur=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); for (const part of process.argv[2].split(".")) cur=part==="length" ? cur.length : (Array.isArray(cur) ? cur[Number(part)] : cur[part]); console.log(cur);' "$1" "$2"
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
# actor_key_from_record <assignment-record.json> — recomputes the SAME holderActorKey
|
|
54
|
+
# computeEffectiveState (src/cli/assignment-provider.ts) itself derives: record.actor_key when
|
|
55
|
+
# present (F1 fix, fix-plan iteration 1 — the canonical resolveActor().actor string
|
|
56
|
+
# performLocalClaim/performLocalSupersede now persist), else serializeActor(record.actor) as a
|
|
57
|
+
# back-compat fallback for any pre-F1 record with no actor_key field. The eval never hand-derives
|
|
58
|
+
# (and risks drifting from) the actual holder-key format ensure-session's guard produces for an
|
|
59
|
+
# explicit --actor override.
|
|
60
|
+
actor_key_from_record() {
|
|
61
|
+
node -e '
|
|
62
|
+
const fs = require("fs");
|
|
63
|
+
const { serializeActor } = require(process.argv[1]);
|
|
64
|
+
const rec = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
65
|
+
process.stdout.write(rec.actor_key || serializeActor(rec.actor));
|
|
66
|
+
' "$ACTOR_IDENTITY_HELPER" "$1"
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
# append_liveness_event <root> <subjectId> <actor> <at-iso> [ttlSeconds]
|
|
70
|
+
append_liveness_event() {
|
|
71
|
+
local root="$1" subject="$2" actor="$3" at="$4" ttl="${5:-1800}"
|
|
72
|
+
mkdir -p "$root/liveness"
|
|
73
|
+
node -e '
|
|
74
|
+
const fs = require("fs");
|
|
75
|
+
const evt = { type: "claim", subjectId: process.argv[1], actor: process.argv[2], at: process.argv[3], ttlSeconds: Number(process.argv[4]) };
|
|
76
|
+
fs.appendFileSync(process.argv[5], JSON.stringify(evt) + "\n");
|
|
77
|
+
' "$subject" "$actor" "$at" "$ttl" "$root/liveness/events.jsonl"
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if [[ ! -f "$CLI" ]]; then
|
|
81
|
+
echo "build/src/cli.js not found — run 'npm run build' first" >&2
|
|
82
|
+
exit 1
|
|
83
|
+
fi
|
|
84
|
+
flow_agents_build_ts || { echo "build failed" >&2; exit 1; }
|
|
85
|
+
|
|
86
|
+
echo "=== ensure-session ownership guard (#291) ==="
|
|
87
|
+
|
|
88
|
+
# ─── 1. Actor A claims a free subject via ensure-session (AC5) ─────────────────────────────
|
|
89
|
+
echo "--- 1. free subject: ensure-session establishes a durable claim (AC5) ---"
|
|
90
|
+
|
|
91
|
+
SUBJECT_WORK_ITEM="kontourai/flow-agents#9101"
|
|
92
|
+
SLUG="kontourai-flow-agents-9101"
|
|
93
|
+
|
|
94
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
95
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
96
|
+
--work-item "$SUBJECT_WORK_ITEM" \
|
|
97
|
+
--actor eval-actor-a-session \
|
|
98
|
+
--source-request "Actor A claims a free subject." \
|
|
99
|
+
--summary "Actor A establishes the first claim on a free subject." \
|
|
100
|
+
>"$TMPDIR_EVAL/a-ensure.out" 2>"$TMPDIR_EVAL/a-ensure.err"; then
|
|
101
|
+
pass "actor A's ensure-session on a free subject exits 0 (AC5)"
|
|
102
|
+
else
|
|
103
|
+
fail "actor A's ensure-session on a free subject unexpectedly failed: $(cat "$TMPDIR_EVAL/a-ensure.out" "$TMPDIR_EVAL/a-ensure.err")"
|
|
104
|
+
fi
|
|
105
|
+
|
|
106
|
+
node "$CLI" assignment-provider status \
|
|
107
|
+
--provider local-file --artifact-root "$ARTIFACT_ROOT" --subject-id "$SLUG" \
|
|
108
|
+
> "$TMPDIR_EVAL/status-after-a.json"
|
|
109
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-after-a.json" "assignment.record.status")" == "claimed" ]] && pass "assignment-provider status confirms a durable claim record exists for actor A (AC5)" || fail "assignment-provider status did not show a durable claim record for actor A: $(cat "$TMPDIR_EVAL/status-after-a.json")"
|
|
110
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-after-a.json" "assignment.assignee")" == *"eval-actor-a-session"* ]] && pass "assignment-provider status reports actor A as the claim holder (AC5)" || fail "assignment-provider status did not report actor A as holder: $(cat "$TMPDIR_EVAL/status-after-a.json")"
|
|
111
|
+
|
|
112
|
+
A_RECORD="$ARTIFACT_ROOT/assignment/$SLUG.json"
|
|
113
|
+
[[ -f "$A_RECORD" ]] && pass "ensure-session's free-branch claim wrote assignment/$SLUG.json (AC5)" || fail "ensure-session's free-branch claim did not write assignment/$SLUG.json"
|
|
114
|
+
ACTOR_KEY_A="$(actor_key_from_record "$A_RECORD")"
|
|
115
|
+
[[ -n "$ACTOR_KEY_A" ]] && pass "actor A's serialized actor key was recovered from the on-disk record" || fail "could not recover actor A's serialized actor key from the on-disk record"
|
|
116
|
+
|
|
117
|
+
# ─── 2. Actor B refused on A's fresh claim (AC1), plus two AC9 sanitization variants ────────
|
|
118
|
+
echo "--- 2. fresh other-actor claim refuses entry (AC1); hostile-string sanitization (AC9) ---"
|
|
119
|
+
|
|
120
|
+
NOW_ISO="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
121
|
+
append_liveness_event "$ARTIFACT_ROOT" "$SLUG" "$ACTOR_KEY_A" "$NOW_ISO" 1800
|
|
122
|
+
|
|
123
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
124
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
125
|
+
--work-item "$SUBJECT_WORK_ITEM" \
|
|
126
|
+
--actor eval-actor-b-session \
|
|
127
|
+
--source-request "Actor B attempts to enter A's fresh claim." \
|
|
128
|
+
--summary "Actor B should be refused." \
|
|
129
|
+
>"$TMPDIR_EVAL/b-ensure.out" 2>"$TMPDIR_EVAL/b-ensure.err"; then
|
|
130
|
+
fail "actor B's ensure-session on A's fresh-held subject should have exited nonzero (AC1)"
|
|
131
|
+
else
|
|
132
|
+
pass "actor B's ensure-session on A's fresh-held subject exits nonzero (AC1)"
|
|
133
|
+
fi
|
|
134
|
+
grep -qF "eval-actor-a-session" "$TMPDIR_EVAL/b-ensure.err" && pass "refusal message names actor A's holder identity (AC1)" || fail "refusal message did not name actor A's holder identity: $(cat "$TMPDIR_EVAL/b-ensure.err")"
|
|
135
|
+
grep -qiF "takeover" "$TMPDIR_EVAL/b-ensure.err" && pass "refusal message offers remediation (pick different work or consider a takeover) (AC1)" || fail "refusal message lacked remediation text: $(cat "$TMPDIR_EVAL/b-ensure.err")"
|
|
136
|
+
|
|
137
|
+
B_DIR="$ARTIFACT_ROOT/$SLUG"
|
|
138
|
+
# A's session dir already legitimately exists (A owns it, from section 1) — a refused entry must
|
|
139
|
+
# never leave any trace of actor B's identity in A's own sidecar.
|
|
140
|
+
if grep -qF "eval-actor-b-session" "$B_DIR/state.json" 2>/dev/null; then
|
|
141
|
+
fail "actor B's refused entry unexpectedly left a trace in A's state.json"
|
|
142
|
+
else
|
|
143
|
+
pass "actor B's refused entry left no trace in A's state.json"
|
|
144
|
+
fi
|
|
145
|
+
|
|
146
|
+
# AC9 variant 1: a crafted HOSTILE liveness event (no assignment record at all — the
|
|
147
|
+
# "liveness_claim_present_assignment_lagging" join row) — proves the guard sanitizes a holder
|
|
148
|
+
# actor string sourced directly from an attacker-postable liveness stream, not merely from a
|
|
149
|
+
# well-formed assignment record.
|
|
150
|
+
HOSTILE_LIVENESS_SLUG="ensure-guard-hostile-liveness"
|
|
151
|
+
HOSTILE_LIVENESS_ACTOR=$'hostile-liveness-actor\x1b[31;1mFAKE\x07-holder'
|
|
152
|
+
append_liveness_event "$ARTIFACT_ROOT" "$HOSTILE_LIVENESS_SLUG" "$HOSTILE_LIVENESS_ACTOR" "$NOW_ISO" 1800
|
|
153
|
+
|
|
154
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
155
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
156
|
+
--task-slug "$HOSTILE_LIVENESS_SLUG" \
|
|
157
|
+
--actor eval-actor-hostile-reader \
|
|
158
|
+
--source-request "Reader against a hostile liveness-only holder." \
|
|
159
|
+
--summary "Should refuse, sanitized." \
|
|
160
|
+
>"$TMPDIR_EVAL/hostile-liveness.out" 2>"$TMPDIR_EVAL/hostile-liveness.err"; then
|
|
161
|
+
fail "ensure-session against a hostile liveness-only holder should have refused (AC1, AC9)"
|
|
162
|
+
else
|
|
163
|
+
pass "ensure-session against a hostile liveness-only holder refuses (AC1 liveness_claim_present_assignment_lagging row, AC9)"
|
|
164
|
+
fi
|
|
165
|
+
if grep -qF $'\x1b[31;1m' "$TMPDIR_EVAL/hostile-liveness.err" || grep -qF $'\x07' "$TMPDIR_EVAL/hostile-liveness.err"; then
|
|
166
|
+
fail "hostile liveness-event holder actor's raw ANSI/control bytes leaked into the refusal message (AC9)"
|
|
167
|
+
else
|
|
168
|
+
pass "hostile liveness-event holder actor's raw ANSI/control bytes never leak into the refusal message (AC9)"
|
|
169
|
+
fi
|
|
170
|
+
grep -qF "hostile-liveness-actor" "$TMPDIR_EVAL/hostile-liveness.err" && pass "the sanitized (non-control-byte) portion of the hostile holder actor string still appears in the refusal message (AC9)" || fail "sanitized portion of the hostile holder actor string was unexpectedly dropped entirely: $(cat "$TMPDIR_EVAL/hostile-liveness.err")"
|
|
171
|
+
|
|
172
|
+
# AC9 variant 2: hostile-effective-state.json (github-style hostile assignee string) via
|
|
173
|
+
# --effective-state-json — the escape hatch for non-local-file providers (Conflict #5).
|
|
174
|
+
HOSTILE_EFFECTIVE_SLUG="ensure-guard-hostile-effective-state"
|
|
175
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
176
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
177
|
+
--task-slug "$HOSTILE_EFFECTIVE_SLUG" \
|
|
178
|
+
--actor eval-actor-hostile-reader-2 \
|
|
179
|
+
--effective-state-json "$HOSTILE_EFFECTIVE_STATE_FIXTURE" \
|
|
180
|
+
--source-request "Reader against a hostile precomputed effective-state fixture." \
|
|
181
|
+
--summary "Should refuse, sanitized." \
|
|
182
|
+
>"$TMPDIR_EVAL/hostile-effective.out" 2>"$TMPDIR_EVAL/hostile-effective.err"; then
|
|
183
|
+
fail "ensure-session --effective-state-json against the hostile fixture should have refused (AC3, AC9)"
|
|
184
|
+
else
|
|
185
|
+
pass "ensure-session --effective-state-json against the hostile fixture refuses (human-held, AC3, AC9)"
|
|
186
|
+
fi
|
|
187
|
+
if grep -qF $'\x1b[31;1m' "$TMPDIR_EVAL/hostile-effective.err" || grep -qF $'\x07' "$TMPDIR_EVAL/hostile-effective.err"; then
|
|
188
|
+
fail "hostile --effective-state-json assignee's raw ANSI/control bytes leaked into the refusal message (AC9)"
|
|
189
|
+
else
|
|
190
|
+
pass "hostile --effective-state-json assignee's raw ANSI/control bytes never leak into the refusal message (AC9)"
|
|
191
|
+
fi
|
|
192
|
+
grep -qF "hostile-assignee" "$TMPDIR_EVAL/hostile-effective.err" && pass "the sanitized portion of the hostile --effective-state-json assignee string still appears in the refusal message (AC9)" || fail "sanitized portion of the hostile --effective-state-json assignee string was unexpectedly dropped entirely: $(cat "$TMPDIR_EVAL/hostile-effective.err")"
|
|
193
|
+
|
|
194
|
+
# ─── 3. Actor A re-entering its own subject succeeds (AC4) ─────────────────────────────────
|
|
195
|
+
echo "--- 3. self re-entry succeeds with no spurious refusal (AC4) ---"
|
|
196
|
+
|
|
197
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
198
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
199
|
+
--work-item "$SUBJECT_WORK_ITEM" \
|
|
200
|
+
--actor eval-actor-a-session \
|
|
201
|
+
--source-request "Actor A resumes its own session." \
|
|
202
|
+
--summary "Actor A resumes." \
|
|
203
|
+
>"$TMPDIR_EVAL/a-reentry.out" 2>"$TMPDIR_EVAL/a-reentry.err"; then
|
|
204
|
+
pass "actor A re-running ensure-session on its own fresh claim succeeds (AC4)"
|
|
205
|
+
else
|
|
206
|
+
fail "actor A's self re-entry unexpectedly refused: $(cat "$TMPDIR_EVAL/a-reentry.out" "$TMPDIR_EVAL/a-reentry.err")"
|
|
207
|
+
fi
|
|
208
|
+
|
|
209
|
+
# ─── 3b. F1 (fix-plan iteration 1, HIGH): cross-tool self-recognition for an EXPLICIT-OVERRIDE
|
|
210
|
+
# actor (FLOW_AGENTS_ACTOR / --actor). This is the reviewer's exact live repro: a claim
|
|
211
|
+
# established via ensure-session under a bare override value must be recognized as self by a
|
|
212
|
+
# DIFFERENT tool invocation (`assignment-provider status --self-actor <bare value>`), and a fresh
|
|
213
|
+
# liveness heartbeat for that same bare value must join against the claim record too. Before the
|
|
214
|
+
# fix, computeEffectiveState keyed everything on serializeActor(record.actor) — a TRIPLE
|
|
215
|
+
# (`explicit-override:<value>:<host>`) for an override actor — while every other tool (including
|
|
216
|
+
# --self-actor here) uses the BARE value, so this cross-tool check failed even though the guard's
|
|
217
|
+
# OWN internal self-check passed (it always re-derives the same wrapped key on both sides).
|
|
218
|
+
echo "--- 3b. cross-tool self-recognition for an explicit-override actor: assignment-provider status --self-actor recognizes an ensure-session claim (F1, fix-plan iteration 1) ---"
|
|
219
|
+
|
|
220
|
+
CANONICAL_WORK_ITEM="kontourai/flow-agents#9103"
|
|
221
|
+
CANONICAL_SLUG="kontourai-flow-agents-9103"
|
|
222
|
+
CANONICAL_ACTOR="canonical-x"
|
|
223
|
+
|
|
224
|
+
if FLOW_AGENTS_ACTOR="$CANONICAL_ACTOR" flow_agents_node "workflow-sidecar" ensure-session \
|
|
225
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
226
|
+
--work-item "$CANONICAL_WORK_ITEM" \
|
|
227
|
+
--source-request "canonical-x claims a free subject via FLOW_AGENTS_ACTOR (no --actor flag)." \
|
|
228
|
+
--summary "canonical-x establishes the first claim." \
|
|
229
|
+
>"$TMPDIR_EVAL/canonical-ensure.out" 2>"$TMPDIR_EVAL/canonical-ensure.err"; then
|
|
230
|
+
pass "FLOW_AGENTS_ACTOR=canonical-x ensure-session on a free subject exits 0 (F1 setup)"
|
|
231
|
+
else
|
|
232
|
+
fail "FLOW_AGENTS_ACTOR=canonical-x ensure-session unexpectedly failed: $(cat "$TMPDIR_EVAL/canonical-ensure.out" "$TMPDIR_EVAL/canonical-ensure.err")"
|
|
233
|
+
fi
|
|
234
|
+
|
|
235
|
+
CANONICAL_RECORD="$ARTIFACT_ROOT/assignment/$CANONICAL_SLUG.json"
|
|
236
|
+
[[ -f "$CANONICAL_RECORD" ]] && pass "canonical-x's claim record exists on disk (F1 setup)" || fail "canonical-x's claim record was not written"
|
|
237
|
+
CANONICAL_ACTOR_KEY_FIELD="$(json_query "$CANONICAL_RECORD" "actor_key")"
|
|
238
|
+
if [[ "$CANONICAL_ACTOR_KEY_FIELD" == "$CANONICAL_ACTOR" ]]; then
|
|
239
|
+
pass "the claim record's actor_key is the canonical BARE value 'canonical-x', not a serialized explicit-override triple (F1)"
|
|
240
|
+
else
|
|
241
|
+
fail "the claim record's actor_key was not the expected bare canonical value: got '$CANONICAL_ACTOR_KEY_FIELD'"
|
|
242
|
+
fi
|
|
243
|
+
|
|
244
|
+
# (a) assignment-provider status --self-actor canonical-x (the BARE value, exactly what
|
|
245
|
+
# `liveness whoami` / `liveness claim --actor` / pull-work's --self-actor would all use) must
|
|
246
|
+
# recognize this claim as self -- effective_state:held, reason:self_is_holder, NOT reclaimable.
|
|
247
|
+
node "$CLI" assignment-provider status \
|
|
248
|
+
--provider local-file --artifact-root "$ARTIFACT_ROOT" --subject-id "$CANONICAL_SLUG" \
|
|
249
|
+
--liveness-events-json <(echo '[]') \
|
|
250
|
+
--self-actor "$CANONICAL_ACTOR" \
|
|
251
|
+
> "$TMPDIR_EVAL/status-canonical-self.json"
|
|
252
|
+
CANONICAL_SELF_STATE="$(json_query "$TMPDIR_EVAL/status-canonical-self.json" "effective.effective_state")"
|
|
253
|
+
CANONICAL_SELF_REASON="$(json_query "$TMPDIR_EVAL/status-canonical-self.json" "effective.reason")"
|
|
254
|
+
if [[ "$CANONICAL_SELF_STATE" == "held" && "$CANONICAL_SELF_REASON" == "self_is_holder" ]]; then
|
|
255
|
+
pass "assignment-provider status --self-actor canonical-x (bare value) recognizes the ensure-session claim as self: held/self_is_holder, not reclaimable (F1 -- the reviewer's exact repro, now fixed)"
|
|
256
|
+
else
|
|
257
|
+
fail "assignment-provider status --self-actor canonical-x did NOT recognize the claim as self: effective_state=$CANONICAL_SELF_STATE reason=$CANONICAL_SELF_REASON (expected held/self_is_holder): $(cat "$TMPDIR_EVAL/status-canonical-self.json")"
|
|
258
|
+
fi
|
|
259
|
+
|
|
260
|
+
# (b) symmetric case: a FRESH liveness heartbeat for canonical-x (the bare value, exactly what a
|
|
261
|
+
# real heartbeat/liveness-claim event would record as `actor`) must join against the SAME record
|
|
262
|
+
# via fresh_liveness_heartbeat, with a DIFFERENT (or no) --self-actor -- proving the join itself
|
|
263
|
+
# matches on the canonical actor_key, not merely the redundant self-check.
|
|
264
|
+
CANONICAL_HEARTBEAT_NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
265
|
+
append_liveness_event "$ARTIFACT_ROOT" "$CANONICAL_SLUG" "$CANONICAL_ACTOR" "$CANONICAL_HEARTBEAT_NOW" 1800
|
|
266
|
+
|
|
267
|
+
node "$CLI" assignment-provider status \
|
|
268
|
+
--provider local-file --artifact-root "$ARTIFACT_ROOT" --subject-id "$CANONICAL_SLUG" \
|
|
269
|
+
--liveness-stream "$ARTIFACT_ROOT/liveness/events.jsonl" \
|
|
270
|
+
--self-actor "eval-actor-some-other-reader" \
|
|
271
|
+
> "$TMPDIR_EVAL/status-canonical-heartbeat.json"
|
|
272
|
+
CANONICAL_HEARTBEAT_STATE="$(json_query "$TMPDIR_EVAL/status-canonical-heartbeat.json" "effective.effective_state")"
|
|
273
|
+
CANONICAL_HEARTBEAT_REASON="$(json_query "$TMPDIR_EVAL/status-canonical-heartbeat.json" "effective.reason")"
|
|
274
|
+
if [[ "$CANONICAL_HEARTBEAT_STATE" == "held" && "$CANONICAL_HEARTBEAT_REASON" == "fresh_liveness_heartbeat" ]]; then
|
|
275
|
+
pass "a fresh liveness heartbeat for canonical-x (bare value) joins against the ensure-session claim via fresh_liveness_heartbeat, under a DIFFERENT self-actor (F1 -- proves the assignment <-> liveness join itself matches, not just the self-check)"
|
|
276
|
+
else
|
|
277
|
+
fail "the fresh liveness heartbeat for canonical-x did not join against the claim: effective_state=$CANONICAL_HEARTBEAT_STATE reason=$CANONICAL_HEARTBEAT_REASON (expected held/fresh_liveness_heartbeat): $(cat "$TMPDIR_EVAL/status-canonical-heartbeat.json")"
|
|
278
|
+
fi
|
|
279
|
+
|
|
280
|
+
# ─── 4. Stale (reclaimable) claim: refuse without the flag, succeed with it (AC2) ──────────
|
|
281
|
+
echo "--- 4. reclaimable (stale) claim: refuse without --supersede-stale, succeed with it (AC2) ---"
|
|
282
|
+
|
|
283
|
+
STALE_SLUG="ensure-guard-stale-takeover"
|
|
284
|
+
flow_agents_node "workflow-sidecar" ensure-session \
|
|
285
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
286
|
+
--task-slug "$STALE_SLUG" \
|
|
287
|
+
--actor eval-actor-stale-holder \
|
|
288
|
+
--source-request "Establish the original (soon-to-be-stale) claim." \
|
|
289
|
+
--summary "Original holder." \
|
|
290
|
+
>"$TMPDIR_EVAL/stale-setup.out" 2>"$TMPDIR_EVAL/stale-setup.err"
|
|
291
|
+
STALE_RECORD="$ARTIFACT_ROOT/assignment/$STALE_SLUG.json"
|
|
292
|
+
ACTOR_KEY_STALE_HOLDER="$(actor_key_from_record "$STALE_RECORD")"
|
|
293
|
+
# A liveness event whose `at` is well past ttlSeconds (1800s) relative to real wall-clock now —
|
|
294
|
+
# matches evals/fixtures/assignment-provider/liveness-stale.json's "well in the past" convention.
|
|
295
|
+
append_liveness_event "$ARTIFACT_ROOT" "$STALE_SLUG" "$ACTOR_KEY_STALE_HOLDER" "2026-06-01T10:00:00Z" 1800
|
|
296
|
+
|
|
297
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
298
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
299
|
+
--task-slug "$STALE_SLUG" \
|
|
300
|
+
--actor eval-actor-b-session \
|
|
301
|
+
--source-request "Actor B attempts a stale takeover without --supersede-stale." \
|
|
302
|
+
--summary "Should refuse, naming the flag." \
|
|
303
|
+
>"$TMPDIR_EVAL/stale-refuse.out" 2>"$TMPDIR_EVAL/stale-refuse.err"; then
|
|
304
|
+
fail "ensure-session on a reclaimable (stale) subject without --supersede-stale should have refused (AC2)"
|
|
305
|
+
else
|
|
306
|
+
pass "ensure-session on a reclaimable (stale) subject without --supersede-stale refuses (AC2)"
|
|
307
|
+
fi
|
|
308
|
+
grep -qF -- "--supersede-stale" "$TMPDIR_EVAL/stale-refuse.err" && pass "stale-claim refusal names --supersede-stale explicitly (AC2)" || fail "stale-claim refusal did not name --supersede-stale: $(cat "$TMPDIR_EVAL/stale-refuse.err")"
|
|
309
|
+
|
|
310
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
311
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
312
|
+
--task-slug "$STALE_SLUG" \
|
|
313
|
+
--actor eval-actor-b-session \
|
|
314
|
+
--supersede-stale \
|
|
315
|
+
--source-request "Actor B takes over the stale claim explicitly." \
|
|
316
|
+
--summary "Explicit takeover." \
|
|
317
|
+
>"$TMPDIR_EVAL/stale-supersede.out" 2>"$TMPDIR_EVAL/stale-supersede.err"; then
|
|
318
|
+
pass "ensure-session --supersede-stale on a reclaimable (stale) subject succeeds (AC2)"
|
|
319
|
+
else
|
|
320
|
+
fail "ensure-session --supersede-stale on a reclaimable (stale) subject unexpectedly failed: $(cat "$TMPDIR_EVAL/stale-supersede.out" "$TMPDIR_EVAL/stale-supersede.err")"
|
|
321
|
+
fi
|
|
322
|
+
|
|
323
|
+
node "$CLI" assignment-provider status \
|
|
324
|
+
--provider local-file --artifact-root "$ARTIFACT_ROOT" --subject-id "$STALE_SLUG" \
|
|
325
|
+
> "$TMPDIR_EVAL/status-after-supersede.json"
|
|
326
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-after-supersede.json" "assignment.assignee")" == *"eval-actor-b-session"* ]] && pass "assignment-provider status confirms actor B is the new holder after --supersede-stale takeover (AC2)" || fail "assignment-provider status did not confirm actor B as the new holder: $(cat "$TMPDIR_EVAL/status-after-supersede.json")"
|
|
327
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-after-supersede.json" "assignment.record.audit_trail.length")" == "2" ]] && pass "supersede takeover appended a second audit_trail entry (claim + supersede) (AC2)" || fail "supersede takeover did not append a second audit_trail entry: $(cat "$TMPDIR_EVAL/status-after-supersede.json")"
|
|
328
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-after-supersede.json" "assignment.record.audit_trail.1.transition")" == "supersede" ]] && pass "audit trail records the supersede transition from the stale-claim takeover (AC2)" || fail "audit trail did not record a supersede transition: $(cat "$TMPDIR_EVAL/status-after-supersede.json")"
|
|
329
|
+
|
|
330
|
+
# ─── 5. Human-held: never auto-reclaims (AC3) ──────────────────────────────────────────────
|
|
331
|
+
echo "--- 5. human-held subject: ask-first refusal, never auto-reclaims (AC3) ---"
|
|
332
|
+
|
|
333
|
+
HUMAN_SLUG="ensure-guard-human-held"
|
|
334
|
+
mkdir -p "$ARTIFACT_ROOT/assignment"
|
|
335
|
+
cat > "$ARTIFACT_ROOT/assignment/$HUMAN_SLUG.json" <<HUMANEOF
|
|
336
|
+
{
|
|
337
|
+
"schema_version": "1.0",
|
|
338
|
+
"role": "AssignmentClaimRecord",
|
|
339
|
+
"subject_id": "$HUMAN_SLUG",
|
|
340
|
+
"actor": {
|
|
341
|
+
"runtime": "github",
|
|
342
|
+
"session_id": "n-a",
|
|
343
|
+
"host": "n-a",
|
|
344
|
+
"human": "alice-human"
|
|
345
|
+
},
|
|
346
|
+
"claimed_at": "2026-06-15T09:00:00Z",
|
|
347
|
+
"ttl_seconds": 1800,
|
|
348
|
+
"branch": "main",
|
|
349
|
+
"artifact_dir": "$HUMAN_SLUG",
|
|
350
|
+
"status": "claimed",
|
|
351
|
+
"audit_trail": [
|
|
352
|
+
{ "at": "2026-06-15T09:00:00Z", "transition": "claim", "from_actor": null, "to_actor": { "runtime": "github", "session_id": "n-a", "host": "n-a", "human": "alice-human" }, "reason": "human assignment" }
|
|
353
|
+
]
|
|
354
|
+
}
|
|
355
|
+
HUMANEOF
|
|
356
|
+
HUMAN_RECORD_BEFORE="$(cat "$ARTIFACT_ROOT/assignment/$HUMAN_SLUG.json")"
|
|
357
|
+
|
|
358
|
+
if flow_agents_node "workflow-sidecar" ensure-session \
|
|
359
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
360
|
+
--task-slug "$HUMAN_SLUG" \
|
|
361
|
+
--actor eval-actor-c-session \
|
|
362
|
+
--source-request "Any actor attempts entry on a human-held subject." \
|
|
363
|
+
--summary "Should refuse, ask-first." \
|
|
364
|
+
>"$TMPDIR_EVAL/human-held.out" 2>"$TMPDIR_EVAL/human-held.err"; then
|
|
365
|
+
fail "ensure-session on a human-held subject should have refused (AC3)"
|
|
366
|
+
else
|
|
367
|
+
pass "ensure-session on a human-held subject refuses (AC3)"
|
|
368
|
+
fi
|
|
369
|
+
grep -qiF "human" "$TMPDIR_EVAL/human-held.err" && pass "human-held refusal message references the human assignment (AC3)" || fail "human-held refusal message did not reference the human assignment: $(cat "$TMPDIR_EVAL/human-held.err")"
|
|
370
|
+
grep -qiF "confirm" "$TMPDIR_EVAL/human-held.err" && pass "human-held refusal message asks for confirmation before proceeding, never auto-reclaims (AC3)" || fail "human-held refusal message lacked ask-first remediation: $(cat "$TMPDIR_EVAL/human-held.err")"
|
|
371
|
+
[[ ! -d "$ARTIFACT_ROOT/$HUMAN_SLUG" ]] && pass "human-held refusal never created a session directory (no auto-reclaim side effect) (AC3)" || fail "human-held refusal unexpectedly created a session directory"
|
|
372
|
+
HUMAN_RECORD_AFTER="$(cat "$ARTIFACT_ROOT/assignment/$HUMAN_SLUG.json")"
|
|
373
|
+
[[ "$HUMAN_RECORD_BEFORE" == "$HUMAN_RECORD_AFTER" ]] && pass "human-held assignment record is byte-identical after the refused attempt (never auto-reclaimed) (AC3)" || fail "human-held assignment record was mutated by a refused ensure-session attempt"
|
|
374
|
+
|
|
375
|
+
# ─── 6. Concurrency: exactly one holder, no corrupted/partial record (AC6) ─────────────────
|
|
376
|
+
echo "--- 6. genuine concurrent ensure-session race: exactly one holder wins (AC6) ---"
|
|
377
|
+
|
|
378
|
+
RACE_SLUG="ensure-guard-race"
|
|
379
|
+
(
|
|
380
|
+
flow_agents_node "workflow-sidecar" ensure-session \
|
|
381
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
382
|
+
--task-slug "$RACE_SLUG" \
|
|
383
|
+
--actor eval-race-actor-a \
|
|
384
|
+
--source-request "Race participant A." \
|
|
385
|
+
--summary "Race participant A." \
|
|
386
|
+
>"$TMPDIR_EVAL/race-a.out" 2>"$TMPDIR_EVAL/race-a.err"
|
|
387
|
+
echo $? > "$TMPDIR_EVAL/race-a.rc"
|
|
388
|
+
) &
|
|
389
|
+
RACE_PID_A=$!
|
|
390
|
+
(
|
|
391
|
+
flow_agents_node "workflow-sidecar" ensure-session \
|
|
392
|
+
--artifact-root "$ARTIFACT_ROOT" \
|
|
393
|
+
--task-slug "$RACE_SLUG" \
|
|
394
|
+
--actor eval-race-actor-b \
|
|
395
|
+
--source-request "Race participant B." \
|
|
396
|
+
--summary "Race participant B." \
|
|
397
|
+
>"$TMPDIR_EVAL/race-b.out" 2>"$TMPDIR_EVAL/race-b.err"
|
|
398
|
+
echo $? > "$TMPDIR_EVAL/race-b.rc"
|
|
399
|
+
) &
|
|
400
|
+
RACE_PID_B=$!
|
|
401
|
+
wait "$RACE_PID_A" "$RACE_PID_B"
|
|
402
|
+
|
|
403
|
+
RACE_RC_A="$(cat "$TMPDIR_EVAL/race-a.rc")"
|
|
404
|
+
RACE_RC_B="$(cat "$TMPDIR_EVAL/race-b.rc")"
|
|
405
|
+
if { [[ "$RACE_RC_A" -eq 0 && "$RACE_RC_B" -ne 0 ]] || [[ "$RACE_RC_A" -ne 0 && "$RACE_RC_B" -eq 0 ]]; }; then
|
|
406
|
+
pass "genuine concurrent ensure-session race: exactly one process wins (rc_a=$RACE_RC_A rc_b=$RACE_RC_B) (AC6)"
|
|
407
|
+
else
|
|
408
|
+
fail "genuine concurrent ensure-session race: exactly one process should win (rc_a=$RACE_RC_A rc_b=$RACE_RC_B) (AC6)"
|
|
409
|
+
fi
|
|
410
|
+
|
|
411
|
+
RACE_RECORD="$ARTIFACT_ROOT/assignment/$RACE_SLUG.json"
|
|
412
|
+
if node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$RACE_RECORD" 2>/dev/null; then
|
|
413
|
+
pass "race winner's on-disk assignment record is valid, uncorrupted JSON (AC6)"
|
|
414
|
+
else
|
|
415
|
+
fail "race winner's on-disk assignment record is missing or corrupted JSON (AC6)"
|
|
416
|
+
fi
|
|
417
|
+
node "$CLI" assignment-provider status \
|
|
418
|
+
--provider local-file --artifact-root "$ARTIFACT_ROOT" --subject-id "$RACE_SLUG" \
|
|
419
|
+
> "$TMPDIR_EVAL/status-race.json"
|
|
420
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-race.json" "assignment.record.audit_trail.length")" == "1" ]] && pass "race winner's on-disk record has exactly one audit_trail entry (no lost/merged/double claim) (AC6)" || fail "race winner's on-disk record does not have exactly one audit_trail entry: $(cat "$TMPDIR_EVAL/status-race.json")"
|
|
421
|
+
|
|
422
|
+
if [[ "$RACE_RC_A" -eq 0 ]]; then
|
|
423
|
+
EXPECTED_RACE_HOLDER="eval-race-actor-a"
|
|
424
|
+
else
|
|
425
|
+
EXPECTED_RACE_HOLDER="eval-race-actor-b"
|
|
426
|
+
fi
|
|
427
|
+
[[ "$(json_query "$TMPDIR_EVAL/status-race.json" "assignment.assignee")" == *"$EXPECTED_RACE_HOLDER"* ]] && pass "on-disk record holder matches the process that actually exited 0 (no silent overwrite by the loser) (AC6)" || fail "on-disk record holder does not match the winning process: $(cat "$TMPDIR_EVAL/status-race.json")"
|
|
428
|
+
|
|
429
|
+
echo ""
|
|
430
|
+
if [[ "$errors" -eq 0 ]]; then
|
|
431
|
+
echo "test_ensure_session_ownership_guard: all checks passed."
|
|
432
|
+
else
|
|
433
|
+
echo "test_ensure_session_ownership_guard: $errors check(s) failed."
|
|
434
|
+
fi
|
|
435
|
+
exit "$errors"
|
|
@@ -706,6 +706,42 @@ else
|
|
|
706
706
|
_fail "AC1.26: redirect > delivery/trust.bundle NOT blocked (exit=$prot_exit)"
|
|
707
707
|
fi
|
|
708
708
|
|
|
709
|
+
echo ""
|
|
710
|
+
echo "--- AC1.26b: #379 per-session Write to delivery/<slug>/trust.bundle BLOCKED ---"
|
|
711
|
+
set +e
|
|
712
|
+
prot_out=$(echo '{"tool_name":"Write","tool_input":{"path":"/repo/delivery/i379-delivery-paths/trust.bundle"}}' | node "$PROT" 2>&1)
|
|
713
|
+
prot_exit=$?
|
|
714
|
+
set -e
|
|
715
|
+
if [ "$prot_exit" -eq 2 ] && echo "$prot_out" | grep -q "BLOCKED"; then
|
|
716
|
+
_pass "AC1.26b: Write to delivery/<slug>/trust.bundle blocked (exit 2) — #379 forgery surface moved with the write path"
|
|
717
|
+
else
|
|
718
|
+
_fail "AC1.26b: Write to delivery/<slug>/trust.bundle NOT blocked (exit=$prot_exit, out=$prot_out)"
|
|
719
|
+
fi
|
|
720
|
+
|
|
721
|
+
echo ""
|
|
722
|
+
echo "--- AC1.26c: #379 per-session cp to delivery/<slug>/trust.checkpoint.json BLOCKED ---"
|
|
723
|
+
set +e
|
|
724
|
+
prot_out=$(echo '{"tool_name":"Bash","tool_input":{"command":"cp forged.json delivery/some-session/trust.checkpoint.json"}}' | node "$PROT" 2>&1)
|
|
725
|
+
prot_exit=$?
|
|
726
|
+
set -e
|
|
727
|
+
if [ "$prot_exit" -eq 2 ] && echo "$prot_out" | grep -q "BLOCKED"; then
|
|
728
|
+
_pass "AC1.26c: cp to delivery/<slug>/trust.checkpoint.json blocked (exit 2)"
|
|
729
|
+
else
|
|
730
|
+
_fail "AC1.26c: cp to delivery/<slug>/trust.checkpoint.json NOT blocked (exit=$prot_exit, out=$prot_out)"
|
|
731
|
+
fi
|
|
732
|
+
|
|
733
|
+
echo ""
|
|
734
|
+
echo "--- AC1.26d: #379 per-session redirect > delivery/<slug>/trust.bundle BLOCKED ---"
|
|
735
|
+
set +e
|
|
736
|
+
prot_out=$(echo '{"tool_name":"Bash","tool_input":{"command":"echo {} > delivery/some-session/trust.bundle"}}' | node "$PROT" 2>&1)
|
|
737
|
+
prot_exit=$?
|
|
738
|
+
set -e
|
|
739
|
+
if [ "$prot_exit" -eq 2 ] && echo "$prot_out" | grep -q "BLOCKED"; then
|
|
740
|
+
_pass "AC1.26d: redirect > delivery/<slug>/trust.bundle blocked (exit 2)"
|
|
741
|
+
else
|
|
742
|
+
_fail "AC1.26d: redirect > delivery/<slug>/trust.bundle NOT blocked (exit=$prot_exit)"
|
|
743
|
+
fi
|
|
744
|
+
|
|
709
745
|
echo ""
|
|
710
746
|
echo "--- AC1.27: cp x src/foo.ts ALLOWED (no over-block on normal copy) ---"
|
|
711
747
|
set +e
|
|
@@ -829,6 +865,13 @@ mkdir -p "$ISO_DIR/repo/.kontourai/flow-agents/surftest"
|
|
|
829
865
|
mkdir -p "$ISO_DIR/lib"
|
|
830
866
|
cp "$GATE" "$ISO_DIR/stop-goal-fit.js"
|
|
831
867
|
cp "$ROOT/scripts/hooks/lib/local-artifact-paths.js" "$ISO_DIR/lib/"
|
|
868
|
+
# #291: stop-goal-fit.js now also requires scripts/hooks/lib/actor-identity.js and
|
|
869
|
+
# scripts/hooks/lib/current-pointer.js (the per-actor current.json compat-shim read helper)
|
|
870
|
+
# -- both must be mirrored into this isolated copy too, or the isolated gate crashes on
|
|
871
|
+
# MODULE_NOT_FOUND before it ever reaches the surface-unavailable fail-closed path this
|
|
872
|
+
# section is testing.
|
|
873
|
+
cp "$ROOT/scripts/hooks/lib/actor-identity.js" "$ISO_DIR/lib/"
|
|
874
|
+
cp "$ROOT/scripts/hooks/lib/current-pointer.js" "$ISO_DIR/lib/"
|
|
832
875
|
printf '# Repo\n' > "$ISO_DIR/repo/AGENTS.md"
|
|
833
876
|
# Non-terminal session (execution phase, in_progress status)
|
|
834
877
|
printf '%s' '{"schema_version":"1.0","task_slug":"surftest","status":"in_progress","phase":"execution","updated_at":"2026-06-27T00:00:00Z","next_action":{"status":"in_progress","summary":"running"}}' \
|
|
@@ -875,6 +918,9 @@ mkdir -p "$ISO2_DIR/repo/.kontourai/flow-agents/lowtest"
|
|
|
875
918
|
mkdir -p "$ISO2_DIR/lib"
|
|
876
919
|
cp "$GATE" "$ISO2_DIR/stop-goal-fit.js"
|
|
877
920
|
cp "$ROOT/scripts/hooks/lib/local-artifact-paths.js" "$ISO2_DIR/lib/"
|
|
921
|
+
# #291: same rationale as ISO_DIR above -- mirror the two new scripts/hooks/lib dependencies.
|
|
922
|
+
cp "$ROOT/scripts/hooks/lib/actor-identity.js" "$ISO2_DIR/lib/"
|
|
923
|
+
cp "$ROOT/scripts/hooks/lib/current-pointer.js" "$ISO2_DIR/lib/"
|
|
878
924
|
printf '# Repo\n' > "$ISO2_DIR/repo/AGENTS.md"
|
|
879
925
|
printf '%s' '{"schema_version":"1.0","task_slug":"lowtest","status":"in_progress","phase":"execution","updated_at":"2026-06-27T00:00:00Z","next_action":{"status":"in_progress","summary":"running"}}' \
|
|
880
926
|
> "$ISO2_DIR/repo/.kontourai/flow-agents/lowtest/state.json"
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_model_routing_escalation.sh — #376
|
|
3
|
+
# Deliver-loop eval: a cheap-tier delegate's output FAILS a gate, the fix is
|
|
4
|
+
# re-dispatched one tier HIGHER (escalation), and the escalation + every
|
|
5
|
+
# per-delegation role/model decision are recorded on the session artifact in a
|
|
6
|
+
# shape a downstream economics record (#349) can consume.
|
|
7
|
+
#
|
|
8
|
+
# R2/AC2: fail -> escalate -> pass, escalation recorded with the tier it climbed FROM.
|
|
9
|
+
# R3/AC3: a verify/review delegation's role tier is >= the worker tier it checks.
|
|
10
|
+
# R4/AC4: per-delegation role/model land on agents/<id>/events.jsonl, additive.
|
|
11
|
+
set -uo pipefail
|
|
12
|
+
|
|
13
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
14
|
+
source "$ROOT/evals/lib/node.sh"
|
|
15
|
+
|
|
16
|
+
WRITER="workflow-sidecar"
|
|
17
|
+
VALIDATOR="validate-workflow-artifacts"
|
|
18
|
+
TMPDIR_EVAL="$(mktemp -d)"
|
|
19
|
+
errors=0
|
|
20
|
+
cleanup() { rm -rf "$TMPDIR_EVAL"; }
|
|
21
|
+
trap cleanup EXIT
|
|
22
|
+
|
|
23
|
+
_pass() { echo " ✓ $1"; }
|
|
24
|
+
_fail() { echo " ✗ $1"; errors=$((errors + 1)); }
|
|
25
|
+
|
|
26
|
+
AR="$TMPDIR_EVAL/repo/.kontourai/flow-agents"
|
|
27
|
+
SLUG="routing-escalation"
|
|
28
|
+
SDIR="$AR/$SLUG"
|
|
29
|
+
mkdir -p "$AR"
|
|
30
|
+
|
|
31
|
+
ev() { flow_agents_node "$WRITER" record-agent-event --artifact-root "$AR" "$@"; }
|
|
32
|
+
|
|
33
|
+
# ── session ───────────────────────────────────────────────────────────────────
|
|
34
|
+
if flow_agents_node "$WRITER" ensure-session \
|
|
35
|
+
--artifact-root "$AR" --task-slug "$SLUG" \
|
|
36
|
+
--title "Routing escalation" \
|
|
37
|
+
--summary "Cheap tier fails a gate; fix escalates one tier higher." \
|
|
38
|
+
--criterion "Escalation recorded" \
|
|
39
|
+
--timestamp "2026-07-04T00:00:00Z" >"$TMPDIR_EVAL/ensure.out" 2>&1; then
|
|
40
|
+
_pass "ensure-session created the deliver-loop session"
|
|
41
|
+
else
|
|
42
|
+
_fail "ensure-session failed: $(cat "$TMPDIR_EVAL/ensure.out")"
|
|
43
|
+
fi
|
|
44
|
+
|
|
45
|
+
# ── deliver loop: delegate cheap -> gate FAIL -> escalate one tier up -> PASS ──
|
|
46
|
+
# 1. Dispatch a fully-specified mechanical slice at the cheap tier (R4 record).
|
|
47
|
+
ev --agent-id tool-worker-1 --kind delegation --status active \
|
|
48
|
+
--role delegate-mechanical --model "claude-haiku-4-5@anthropic" \
|
|
49
|
+
--summary "mechanical slice: routine config edit" \
|
|
50
|
+
--timestamp "2026-07-04T00:00:01Z" >/dev/null 2>&1 \
|
|
51
|
+
&& _pass "recorded mechanical delegation (delegate-mechanical)" \
|
|
52
|
+
|| _fail "failed to record mechanical delegation"
|
|
53
|
+
|
|
54
|
+
# 2. Verify gate FAILS on that cheap delegate's output.
|
|
55
|
+
ev --agent-id tool-verifier --kind evidence --status fail \
|
|
56
|
+
--role delegate-implementation --model "claude-sonnet-5@anthropic" \
|
|
57
|
+
--summary "verify gate FAILED on tool-worker-1 output" \
|
|
58
|
+
--timestamp "2026-07-04T00:00:02Z" >/dev/null 2>&1 \
|
|
59
|
+
&& _pass "recorded verify gate FAILURE on cheap-tier output" \
|
|
60
|
+
|| _fail "failed to record gate failure"
|
|
61
|
+
|
|
62
|
+
# 3. Escalate: re-dispatch the FIX one tier higher, recording the tier climbed FROM.
|
|
63
|
+
ev --agent-id tool-worker-2 --kind escalation --status active \
|
|
64
|
+
--role delegate-implementation --model "claude-sonnet-5@anthropic" \
|
|
65
|
+
--escalated-from delegate-mechanical \
|
|
66
|
+
--summary "fix re-dispatched one tier higher after gate failure" \
|
|
67
|
+
--timestamp "2026-07-04T00:00:03Z" >/dev/null 2>&1 \
|
|
68
|
+
&& _pass "recorded escalation (delegate-mechanical -> delegate-implementation)" \
|
|
69
|
+
|| _fail "failed to record escalation"
|
|
70
|
+
|
|
71
|
+
# 4. Re-verify PASSES on the escalated fix. Goodhart guard: the verifier tier
|
|
72
|
+
# (delegate-implementation) is >= the worker tier it checks (delegate-implementation).
|
|
73
|
+
ev --agent-id tool-verifier --kind evidence --status pass \
|
|
74
|
+
--role delegate-implementation --model "claude-sonnet-5@anthropic" \
|
|
75
|
+
--summary "verify gate PASSED on escalated fix" \
|
|
76
|
+
--timestamp "2026-07-04T00:00:04Z" >/dev/null 2>&1 \
|
|
77
|
+
&& _pass "recorded verify PASS after escalation" \
|
|
78
|
+
|| _fail "failed to record verify pass"
|
|
79
|
+
|
|
80
|
+
# ── assertions over the recorded session artifact ─────────────────────────────
|
|
81
|
+
node - "$SDIR" <<'NODE'
|
|
82
|
+
const fs = require("fs");
|
|
83
|
+
const path = require("path");
|
|
84
|
+
const sdir = process.argv[2];
|
|
85
|
+
const TIER = { "delegate-mechanical": 1, "delegate-implementation": 2, "delegate-design": 3, "orchestrator": 4 };
|
|
86
|
+
|
|
87
|
+
function readEvents(agent) {
|
|
88
|
+
const f = path.join(sdir, "agents", agent, "events.jsonl");
|
|
89
|
+
if (!fs.existsSync(f)) return [];
|
|
90
|
+
return fs.readFileSync(f, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let bad = 0;
|
|
94
|
+
const say = (ok, msg) => { console.log(` ${ok ? "✓" : "✗"} ${msg}`); if (!ok) bad++; };
|
|
95
|
+
|
|
96
|
+
// R4/AC4: every delegation/escalation event carries a consumable role + model.
|
|
97
|
+
const w1 = readEvents("tool-worker-1");
|
|
98
|
+
const del = w1.find((e) => e.kind === "delegation");
|
|
99
|
+
say(del && del.role === "delegate-mechanical" && del.model === "claude-haiku-4-5@anthropic",
|
|
100
|
+
"R4: mechanical delegation event carries role + model (economics-consumable)");
|
|
101
|
+
|
|
102
|
+
// R4 additive/back-compat: a plain event carries no routing keys.
|
|
103
|
+
const plainRoot = path.join(sdir, "state.json");
|
|
104
|
+
say(fs.existsSync(plainRoot), "session state.json exists (additive change did not break session shape)");
|
|
105
|
+
|
|
106
|
+
// R2/AC2: escalation event recorded, one tier higher, with the tier climbed FROM.
|
|
107
|
+
const esc = readEvents("tool-worker-2").find((e) => e.kind === "escalation");
|
|
108
|
+
say(!!esc, "R2: escalation event recorded");
|
|
109
|
+
say(esc && esc.escalated_from === "delegate-mechanical", "R2: escalation records escalated_from = delegate-mechanical");
|
|
110
|
+
say(esc && TIER[esc.role] === TIER[esc.escalated_from] + 1,
|
|
111
|
+
"R2: escalation climbed exactly one tier up the ladder (mechanical -> implementation)");
|
|
112
|
+
|
|
113
|
+
// R3/AC3: the verifier tier is >= the worker tier it checks (Goodhart guard).
|
|
114
|
+
const verifs = readEvents("tool-verifier").filter((e) => e.role);
|
|
115
|
+
const workerTier = TIER["delegate-implementation"]; // tier of the escalated fix under verification
|
|
116
|
+
const allGuarded = verifs.every((v) => TIER[v.role] >= workerTier);
|
|
117
|
+
say(verifs.length >= 1 && allGuarded, "R3: verify role tier >= worker tier (never a cheaper checker)");
|
|
118
|
+
|
|
119
|
+
// AC4: demonstrate the #349-consumable reduction — price per role across the run.
|
|
120
|
+
const allEvents = [];
|
|
121
|
+
for (const agent of fs.readdirSync(path.join(sdir, "agents"))) {
|
|
122
|
+
allEvents.push(...readEvents(agent));
|
|
123
|
+
}
|
|
124
|
+
const byRole = {};
|
|
125
|
+
for (const e of allEvents) if (e.role) byRole[e.role] = (byRole[e.role] || 0) + 1;
|
|
126
|
+
say(Object.keys(byRole).length >= 2 && byRole["delegate-mechanical"] >= 1 && byRole["delegate-implementation"] >= 1,
|
|
127
|
+
`AC4: routing records reduce to a per-role economics shape: ${JSON.stringify(byRole)}`);
|
|
128
|
+
|
|
129
|
+
process.exit(bad === 0 ? 0 : 1);
|
|
130
|
+
NODE
|
|
131
|
+
if [[ $? -eq 0 ]]; then _pass "session artifact routing/escalation/Goodhart assertions passed"; else _fail "session artifact assertions failed"; fi
|
|
132
|
+
|
|
133
|
+
# ── additive shape must still validate ────────────────────────────────────────
|
|
134
|
+
if flow_agents_node "$VALIDATOR" "$SDIR" >"$TMPDIR_EVAL/validate.out" 2>&1; then
|
|
135
|
+
_pass "validate-workflow-artifacts accepts the additive routing fields"
|
|
136
|
+
else
|
|
137
|
+
_fail "validate-workflow-artifacts rejected routing fields: $(cat "$TMPDIR_EVAL/validate.out")"
|
|
138
|
+
fi
|
|
139
|
+
|
|
140
|
+
if [[ "$errors" -eq 0 ]]; then
|
|
141
|
+
echo "Model routing escalation (deliver-loop) integration passed."
|
|
142
|
+
exit 0
|
|
143
|
+
fi
|
|
144
|
+
echo "Model routing escalation integration failed: $errors issue(s)."
|
|
145
|
+
exit 1
|