@kontourai/flow-agents 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +16 -0
- package/CHANGELOG.md +30 -0
- package/build/src/cli/assignment-provider.d.ts +45 -0
- package/build/src/cli/assignment-provider.js +107 -13
- package/build/src/cli/workflow-artifact-cleanup-audit.js +418 -11
- package/build/src/cli/workflow-sidecar.d.ts +238 -4
- package/build/src/cli/workflow-sidecar.js +875 -14
- package/build/src/tools/validate-source-tree.js +3 -2
- package/context/contracts/artifact-contract.md +16 -2
- package/context/contracts/assignment-provider-contract.md +1 -1
- package/context/contracts/execution-contract.md +78 -0
- package/context/scripts/hooks/config-protection.js +11 -4
- package/context/scripts/hooks/stop-goal-fit.js +259 -4
- package/context/scripts/hooks/workflow-steering.js +73 -1
- package/docs/adr/0022-fail-closed-delivery-reconciliation-with-governed-exemptions.md +111 -0
- package/docs/coordination-guide.md +370 -0
- package/docs/decisions/agent-coordination.md +26 -9
- package/docs/decisions/index.md +2 -2
- package/docs/decisions/trust-reconcile.md +42 -9
- package/docs/fixture-ownership.md +3 -2
- package/docs/index.md +4 -0
- package/docs/integrations/flow-agents-console.md +108 -0
- package/docs/integrations/index.md +4 -0
- package/docs/workflow-artifact-lifecycle.md +38 -1
- package/evals/ci/antigaming-suite.sh +1 -0
- package/evals/ci/run-baseline.sh +8 -0
- package/evals/fixtures/reconcile-preflight/disputed-critique-unsuperseded.json +48 -0
- package/evals/fixtures/reconcile-preflight/standalone-disputed-session-local.json +59 -0
- package/evals/integration/test_checkpoint_signing.sh +14 -5
- package/evals/integration/test_ci_actor_identity.sh +221 -0
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_gate_lockdown.sh +36 -0
- package/evals/integration/test_model_routing_escalation.sh +145 -0
- package/evals/integration/test_publish_delivery.sh +73 -8
- package/evals/integration/test_reconcile_preflight.sh +304 -0
- package/evals/integration/test_stop_hook_release.sh +552 -0
- package/evals/integration/test_takeover_protocol.sh +340 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +261 -0
- package/evals/integration/test_verify_hold.sh +910 -0
- package/evals/integration/test_veritas_governance_kit.sh +257 -0
- package/evals/integration/test_workflow_artifact_cleanup_audit.sh +575 -3
- package/evals/run.sh +14 -0
- package/evals/static/test_model_routing_hints.sh +107 -0
- package/kits/builder/skills/builder-shape/SKILL.md +10 -0
- package/kits/builder/skills/continue-work/SKILL.md +2 -0
- package/kits/builder/skills/deliver/SKILL.md +125 -11
- package/kits/builder/skills/design-probe/SKILL.md +10 -0
- package/kits/builder/skills/execute-plan/SKILL.md +13 -0
- package/kits/builder/skills/fix-bug/SKILL.md +17 -0
- package/kits/builder/skills/idea-to-backlog/SKILL.md +10 -0
- package/kits/builder/skills/plan-work/SKILL.md +9 -0
- package/kits/builder/skills/pull-work/SKILL.md +22 -2
- package/kits/builder/skills/review-work/SKILL.md +11 -0
- package/kits/builder/skills/tdd-workflow/SKILL.md +17 -0
- package/kits/builder/skills/verify-work/SKILL.md +11 -0
- package/kits/knowledge/adapters/default-store/index.js +56 -15
- package/kits/knowledge/adapters/flow-runner/index.js +912 -16
- package/kits/knowledge/adapters/obsidian-store/index.js +29 -11
- package/kits/knowledge/adapters/shared/codec.js +124 -0
- package/kits/knowledge/docs/store-contract.md +405 -3
- package/kits/knowledge/evals/audit-freshness/suite.test.js +92 -1
- package/kits/knowledge/evals/consolidate-incremental/suite.test.js +494 -0
- package/kits/knowledge/evals/consolidation/suite.test.js +1 -1
- package/kits/knowledge/evals/contract-suite/suite.test.js +36 -0
- package/kits/knowledge/evals/freshness/suite.test.js +339 -0
- package/kits/knowledge/evals/inbound-references/suite.test.js +351 -0
- package/kits/knowledge/evals/retirement/suite.test.js +1 -1
- package/kits/knowledge/evals/supersede-propagation/suite.test.js +384 -0
- package/kits/veritas-governance/docs/README.md +81 -3
- package/kits/veritas-governance/fixtures/exemption/approved.trust-bundle.json +74 -0
- package/kits/veritas-governance/fixtures/exemption/not-approved.trust-bundle.json +74 -0
- package/kits/veritas-governance/flows/exemption-issuance.flow.json +35 -0
- package/kits/veritas-governance/kit.json +5 -0
- package/package.json +1 -1
- package/schemas/workflow-handoff.schema.json +6 -0
- package/scripts/ci/mint-attestation.js +33 -6
- package/scripts/ci/trust-reconcile.js +222 -279
- package/scripts/hooks/config-protection.js +11 -4
- package/scripts/hooks/lib/actor-identity.js +82 -0
- package/scripts/hooks/stop-goal-fit.js +259 -4
- package/scripts/hooks/workflow-steering.js +73 -1
- package/scripts/lib/reconcile-shape.js +381 -0
- package/src/cli/assignment-provider.ts +122 -13
- package/src/cli/workflow-artifact-cleanup-audit.ts +483 -10
- package/src/cli/workflow-sidecar.ts +965 -14
- package/src/tools/validate-source-tree.ts +3 -2
package/docs/index.md
CHANGED
|
@@ -118,6 +118,10 @@ Use fix-bug. Reproduce the problem, diagnose root cause, implement the fix, and
|
|
|
118
118
|
<strong>System Guidebook</strong>
|
|
119
119
|
<span>The plain-language map of how Flow Agents is assembled and how it should feel to use.</span>
|
|
120
120
|
</a>
|
|
121
|
+
<a class="doc-card" href="coordination-guide.html">
|
|
122
|
+
<strong>Parallel-Session Coordination</strong>
|
|
123
|
+
<span>Run many sessions against one repo without collisions: the two-stream model (liveness + assignment), the four guard points, the one hard publish fence, and how delivery defends itself.</span>
|
|
124
|
+
</a>
|
|
121
125
|
<a class="doc-card" href="skills-map.html">
|
|
122
126
|
<strong>Workflow Map</strong>
|
|
123
127
|
<span>See the core skills, gates, artifacts, and route-back behavior.</span>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Flow Agents × Kontour Console
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# Flow Agents × Kontour Console
|
|
6
|
+
|
|
7
|
+
Flow Agents' [coordination substrate](../coordination-guide.md) is **local-first and complete on a
|
|
8
|
+
single machine** — the local liveness stream plus the GitHub `AssignmentProvider` give you full
|
|
9
|
+
parallel-session safety with no server. The **Kontour Console** (`console.kontourai.io`) is the
|
|
10
|
+
**optional fleet tier** layered on top: it never becomes the authority for coordination; it *relays,
|
|
11
|
+
projects, and eventually arbitrates* across machines and teammates.
|
|
12
|
+
|
|
13
|
+
> This is the design boundary, stated plainly: **single-machine parallelism is fully unlocked
|
|
14
|
+
> locally, for free.** Multi-machine fleets, cross-teammate visibility, human oversight, history, and
|
|
15
|
+
> team-level insight are what the hosted tier adds — and sells. See [ADR 0021 §7](../adr/0021-assignment-leases-and-stale-claim-takeover.md)
|
|
16
|
+
> for the ratifying decision.
|
|
17
|
+
|
|
18
|
+
**Tracking:** flow-agents epic [#394](https://github.com/kontourai/flow-agents/issues/394) · console
|
|
19
|
+
epic [#123](https://github.com/kontourai/console/issues/123). This doc is the in-repo narrative; the
|
|
20
|
+
issues carry live slice status.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## What the Console already is
|
|
25
|
+
|
|
26
|
+
The Console is a Node/TypeScript monorepo (`console-core` / `console-server` / `console-ui`), backed
|
|
27
|
+
by Postgres (Neon), deployed on Render, running the dogfood instance at `console.kontourai.io`. It
|
|
28
|
+
already ingests and projects agent telemetry:
|
|
29
|
+
|
|
30
|
+
- **Ingest** — `POST /records` accepts telemetry records; auth via `Bearer` / `X-Console-API-Token`
|
|
31
|
+
plus `X-Console-Tenant-Id` (tokens configured through `CONSOLE_AUTH_TOKENS_JSON`).
|
|
32
|
+
- **Projection** — `GET /state` returns an `OperatingState` projection; **SSE** streams live updates.
|
|
33
|
+
- **Economics** — per-session cost/usage views (console #117) and an org-scoped registry (#118).
|
|
34
|
+
|
|
35
|
+
Its multi-tenancy today is **strong at the infrastructure layer** (Postgres primary keys per tenant,
|
|
36
|
+
per-tenant hubs and SSE fan-out) but **weak at the schema layer**: records don't self-identify their
|
|
37
|
+
tenant — tenancy is carried by the request header, not the record body. Closing that gap is part of
|
|
38
|
+
this epic (see *Tenant hardening* below).
|
|
39
|
+
|
|
40
|
+
## The design invariant: relay, never authority
|
|
41
|
+
|
|
42
|
+
Two rules keep the Console from compromising the local-first guarantee:
|
|
43
|
+
|
|
44
|
+
1. **Heartbeats are never written to a provider.** They mirror to the Console relay (the one vantage
|
|
45
|
+
with global heartbeat visibility) but never to GitHub — rate-limit abuse for no benefit. At most a
|
|
46
|
+
claim comment refreshes on phase transitions, doubling as board-level progress.
|
|
47
|
+
2. **The advisory stream stays advisory.** The Console projects the same liveness/assignment signals
|
|
48
|
+
the local substrate computes; in Phase 1 it adds *visibility*, not new authority. Only in Phase 2,
|
|
49
|
+
opt-in, does a Console-backed `AssignmentProvider` make assignment authoritative across machines —
|
|
50
|
+
and even then it implements the exact same contract the GitHub/local providers do.
|
|
51
|
+
|
|
52
|
+
## The phased plan (one epic, two phases)
|
|
53
|
+
|
|
54
|
+
### Phase 1 — read-only fleet insight (the relay)
|
|
55
|
+
|
|
56
|
+
Turn per-repo coordination into a team-wide view, without changing where authority lives.
|
|
57
|
+
|
|
58
|
+
| Slice | Repo | What it does |
|
|
59
|
+
| --- | --- | --- |
|
|
60
|
+
| **ApiSink** ([console #73]) | console | The ingest substrate the relay rides on — the prerequisite. |
|
|
61
|
+
| **Liveness relay** (flow-agents [#295]) | flow-agents | An optional sink: sessions mirror `claim`/`heartbeat`/`release` events to the Console, reusing its idempotent ingest + tenant auth + SSE. Off by default; strictly additive. |
|
|
62
|
+
| **Fleet projection** | console | Active actors, `held`/`reclaimable` subjects, and per-session cost, projected per tenant — the fleet view. |
|
|
63
|
+
| **In-hub janitor** | console | The [coordination janitor](../coordination-guide.md#10-takeover-protocol-forthcoming--294) run centrally: the Console is the one vantage with global heartbeat visibility, so stale-claim reaping for a *fleet* is its natural first cross-machine duty. (Decision: the janitor lives **in the console hub**, not per-client.) |
|
|
64
|
+
|
|
65
|
+
Phase 1 delivers the "see everyone's active sessions, and your own usage in team context" outcome —
|
|
66
|
+
project/team insight views alongside individual ones — while coordination authority stays exactly
|
|
67
|
+
where it is today (local + GitHub).
|
|
68
|
+
|
|
69
|
+
### Phase 2 — authoritative fleet coordination
|
|
70
|
+
|
|
71
|
+
Opt-in, for teams that want cross-machine assignment arbitration:
|
|
72
|
+
|
|
73
|
+
- **Console `AssignmentProvider`** — a third implementation of the [assignment-provider contract](https://github.com/kontourai/flow-agents/blob/main/context/contracts/assignment-provider-contract.md),
|
|
74
|
+
making assignment authoritative across machines (not just within one repo's GitHub board). Same
|
|
75
|
+
`claim`/`release`/`supersede`/`status`/`list` operations; same effective-state join. This is also
|
|
76
|
+
what lets [verify-hold](../coordination-guide.md#8-guard-point-3--the-verify-hold-publish-gate-the-one-hard-fence)
|
|
77
|
+
enforce against a *fleet-wide* holder, not just a local one.
|
|
78
|
+
- **Tenant hardening** (console #98–#100) — close the schema-level gap: put an **explicit `tenant_id`
|
|
79
|
+
on records** (decision) so tenancy is self-describing in the body, not only in the request header,
|
|
80
|
+
hardening the phased auth model end-to-end.
|
|
81
|
+
- **Native init step** — `flow-agents init` gains a first-class, optional Console-integration step:
|
|
82
|
+
configure the relay sink, tenant, and (Phase 2) the Console provider at install time, so joining a
|
|
83
|
+
fleet is a setup choice rather than manual wiring.
|
|
84
|
+
|
|
85
|
+
## How CI fits (paired with #398)
|
|
86
|
+
|
|
87
|
+
The [CI-runtime actor identity tier (#398)](https://github.com/kontourai/flow-agents/issues/398) makes
|
|
88
|
+
CI sessions carry a **stable identity** (e.g. `github-actions:<run-id>` plus `GITHUB_ACTOR`/repo/SHA).
|
|
89
|
+
That has two payoffs here: the [verify-hold gate](../coordination-guide.md#1-the-actor-model--who-am-i)
|
|
90
|
+
upgrades from *advisory* to *enforcing* in CI, and CI becomes an **attributable participant** in the
|
|
91
|
+
fleet — its records land in the same Console projections, so team/project/individual insight views
|
|
92
|
+
include automated work, not just interactive sessions. #398 is the keystone that makes the Console
|
|
93
|
+
fleet view whole.
|
|
94
|
+
|
|
95
|
+
## Dogfooding
|
|
96
|
+
|
|
97
|
+
`console.kontourai.io` runs from the `console` repo; the deployment lives in `console-deploy` (the
|
|
98
|
+
dogfood instance). Flow Agents development itself is intended to run against it — the fleet view of our
|
|
99
|
+
own parallel Builder sessions is both the first real test and the first sales demo of the tier.
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Status
|
|
104
|
+
|
|
105
|
+
Designed and ratified ([ADR 0021 §7](../adr/0021-assignment-leases-and-stale-claim-takeover.md)); the
|
|
106
|
+
local substrate it builds on is complete (see the [coordination guide](../coordination-guide.md)). Phase
|
|
107
|
+
1 begins with console #73 (ApiSink) → flow-agents #295 (relay). This is forward-looking: no Console
|
|
108
|
+
integration ships in the coordination substrate itself — it is strictly the optional tier above it.
|
|
@@ -49,6 +49,10 @@ The <a href="../spec/runtime-hook-surface.html">Runtime Hook Surface spec</a> de
|
|
|
49
49
|
<strong>Knowledge Kit Live Example</strong>
|
|
50
50
|
<span>End-to-end proof of the Knowledge Kit ingest + compile flows against a real Strands agent (OllamaModel / qwen3:1.7b). No API key required. Includes acceptance test with telemetry and provenance assertions.</span>
|
|
51
51
|
</a>
|
|
52
|
+
<a class="doc-card" href="flow-agents-console.html">
|
|
53
|
+
<strong>Flow Agents × Console</strong>
|
|
54
|
+
<span>The optional fleet tier over the local coordination substrate: a liveness relay, team/project insight views, an in-hub janitor, and (Phase 2) a Console-backed AssignmentProvider. Local-first stands; the Console never becomes the authority.</span>
|
|
55
|
+
</a>
|
|
52
56
|
</div>
|
|
53
57
|
|
|
54
58
|
---
|
|
@@ -19,10 +19,47 @@ npm run workflow-artifact-cleanup-audit -- --artifact-root .flow-agents --json
|
|
|
19
19
|
|
|
20
20
|
The default root for local runtime artifacts is `.kontourai/flow-agents`. `.flow-agents` may hold explicit durable Flow Agents config/install state, but it is not a runtime fallback. Move old local sessions with the migration script instead of relying on automatic fallback reads.
|
|
21
21
|
|
|
22
|
-
The command scans immediate workflow directories, skips non-workflow lanes such as `archive/`, and reports active WIP separately from cleanup candidates, terminal done records, active learning follow-ups, and invalid sidecars.
|
|
22
|
+
The command scans immediate workflow directories, skips non-workflow lanes such as `archive/`, and reports active WIP separately from cleanup candidates, terminal done records, active learning follow-ups, and invalid sidecars.
|
|
23
23
|
|
|
24
24
|
Use the Current-State Semantics and Local Retention Policy sections below to interpret each bucket. In particular, learning records with `learning.status: followup_required` or any `routing[].status: open` remain active learning follow-ups until every route is completed, opened elsewhere, deferred with a trigger, accepted, or rejected.
|
|
25
25
|
|
|
26
|
+
## Apply Mode
|
|
27
|
+
|
|
28
|
+
Dry-run classification is the default and remains a zero-side-effect, read-only pass. Pass `--apply` to actually archive-move the sessions the classifier already puts in `cleanup_candidate` (and aged `terminal_done` sessions past the retention window) out of the active listing:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# Dry-run (unchanged; still the default with no flags)
|
|
32
|
+
npm run workflow-artifact-cleanup-audit -- --artifact-root .kontourai/flow-agents
|
|
33
|
+
|
|
34
|
+
# Archive-move cleanup_candidate / aged terminal_done sessions
|
|
35
|
+
npm run workflow-artifact-cleanup-audit -- --artifact-root .kontourai/flow-agents --apply
|
|
36
|
+
|
|
37
|
+
# Also archive-move lifecycle-ambiguous and genuinely-stub invalid sessions (second gate)
|
|
38
|
+
npm run workflow-artifact-cleanup-audit -- --artifact-root .kontourai/flow-agents --apply --apply-ambiguous
|
|
39
|
+
|
|
40
|
+
# Custom retention window and destination
|
|
41
|
+
npm run workflow-artifact-cleanup-audit -- --artifact-root .kontourai/flow-agents \
|
|
42
|
+
--apply --freshness-window-hours 72 --archive-root /path/to/archive
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
**Two-gate model.** `--apply` alone only ever moves `cleanup_candidate` sessions and aged `terminal_done` sessions (the same buckets the manual Session Archival Policy procedure in `context/contracts/artifact-contract.md` already selects by hand) — it never touches anything the classifier calls `active_wip`, `active_learning_followup`, or `invalid`. A session the classifier marks `invalid` with an "unrecognized lifecycle shape" reason (a delivered/accepted session whose shape doesn't fit a known terminal pattern, but is otherwise a real, substantive record) is reported separately as **ambiguous** and is moved only when `--apply-ambiguous` is *also* passed — this is a deliberate second opt-in, not a default, because confirming that kind of straggler is genuinely done (e.g. its referenced PR actually merged) is a human/flow judgment call this tool does not make for you (see `--confirm` below).
|
|
46
|
+
|
|
47
|
+
**Safety rails.** Regardless of flags, `--apply`/`--apply-ambiguous` never touch:
|
|
48
|
+
|
|
49
|
+
1. **Active work** — `active_wip` and `active_learning_followup` sessions are hard-excluded; no flag combination overrides this.
|
|
50
|
+
2. **A session someone else is holding** — if the local liveness stream shows a fresh claim on a session's slug (any actor, not just the current one), that session is skipped and reported as blocked by a held liveness claim, even if the classifier would otherwise archive it.
|
|
51
|
+
3. **Anything too recent** — a session newer than the freshness window (`--freshness-window-hours`, default 48) is never moved, using `state.json`'s `updated_at` as the primary signal; directory modification time is used only as a fallback when `state.json` is missing or unparsable, never preferred over `updated_at` when it is present (bulk worktree operations can reset file mtimes without touching the actual session content, so mtime alone is not a trustworthy freshness signal).
|
|
52
|
+
4. **Infrastructure directories** — per-agent claim/assignment tracking, the runtime adapter's generated output, the skills catalog, and similar structural directories are recognized by their own shape (they carry their own top-level pointer/lock files, i.e. they are themselves a workflow-sidecar-managed root, not a session) and are never treated as sessions to begin with, so a nested runtime tree with real sub-sessions of its own is never flattened into a single misclassified entry or partially swept.
|
|
53
|
+
5. **Substantive sessions with a schema nit** — an `invalid` session that actually has real `state.json` plus `learning.json` and/or `trust.bundle` content is always report-only, even under `--apply --apply-ambiguous`; only a genuinely empty/stub `invalid` directory (no state at all, or state with no sidecar content) is ever eligible for the ambiguous-archive path.
|
|
54
|
+
|
|
55
|
+
**Never delete.** Every move is an archive-move (rename, or copy-then-verify-then-remove-source when a plain rename can't cross filesystems) — the tool has no delete capability of any kind, in any mode.
|
|
56
|
+
|
|
57
|
+
**Where things land.** `--apply` moves matching sessions to `<archive-root>/<date>-<runid>/<slug>/`, where `--archive-root` defaults to a sibling `.kontourai/flow-agents-archive/` next to the artifact root. This is a deliberately *different* destination from the manual, in-tree `<artifact-root>/archive/<slug>/` convention the Session Archival Policy section of `context/contracts/artifact-contract.md` documents for hand-driven moves: giving every automated `--apply` invocation its own dated, run-scoped directory means two same-day runs never collide, and it keeps the artifact root's own scan set immutable while a run is in progress. A human archiving a session by hand may still use the in-tree `archive/<slug>/` path from that section; the two destinations serve the same retirement intent through different (manual vs. automated) procedures — this mode automates the `cleanup_candidate`/aged-`terminal_done` selection that section already specifies, it does not replace the manual path for everything else.
|
|
58
|
+
|
|
59
|
+
**Manifest.** Any `--apply` run that moves at least one session writes `MANIFEST.md` into that run's archive directory, listing each moved session's slug, classification, last-updated timestamp, and the classifier's own reason text. A run that moves nothing writes no manifest and creates no new archive directory. `--confirm <slug>=<evidence>` (repeatable) lets the invoking human or flow record, verbatim, evidence they already independently gathered (for example, that a straggler's referenced PR is confirmed merged) into that slug's manifest row — a slug moved without a matching `--confirm` entry shows `"none recorded"` rather than a fabricated value. This tool makes no network or `gh` calls itself; it only records evidence it is told.
|
|
60
|
+
|
|
61
|
+
This is the same procedure a 2026-07-05 live triage sweep of this repo's own `.kontourai/flow-agents/` tree ran by hand before `--apply` existed (see that sweep's own `MANIFEST.md` for the real cases — stale stubs, aged cleanup candidates, confirmed-merged-PR stragglers, and the infrastructure/schema-nit exclusions above) — the sweep's findings are what shaped the safety rails above; `--apply` automates exactly that already-proven procedure.
|
|
62
|
+
|
|
26
63
|
## Artifact Lanes
|
|
27
64
|
|
|
28
65
|
Use one local non-durable lane under `.kontourai/flow-agents/`:
|
|
@@ -35,6 +35,7 @@ TESTS=(
|
|
|
35
35
|
"evals/integration/test_trust_reconcile_manifest.sh"
|
|
36
36
|
"evals/integration/test_trust_reconcile_mixed_bundle.sh"
|
|
37
37
|
"evals/integration/test_trust_reconcile_negatives.sh"
|
|
38
|
+
"evals/integration/test_reconcile_preflight.sh"
|
|
38
39
|
"evals/integration/test_goal_fit_ghost_session.sh"
|
|
39
40
|
)
|
|
40
41
|
|
package/evals/ci/run-baseline.sh
CHANGED
|
@@ -22,10 +22,14 @@ CHECKS=(
|
|
|
22
22
|
"Workflow sidecar writer integration|bash evals/integration/test_workflow_sidecar_writer.sh"
|
|
23
23
|
"Sidecar field preservation integration|bash evals/integration/test_sidecar_field_preservation.sh"
|
|
24
24
|
"Actor identity resolver integration|bash evals/integration/test_actor_identity.sh"
|
|
25
|
+
"CI-runtime actor identity integration|bash evals/integration/test_ci_actor_identity.sh"
|
|
25
26
|
"Assignment provider local-file integration|bash evals/integration/test_assignment_provider_local_file.sh"
|
|
26
27
|
"Assignment provider github integration|bash evals/integration/test_assignment_provider_github.sh"
|
|
28
|
+
"Stop hook release-with-handoff integration|bash evals/integration/test_stop_hook_release.sh"
|
|
27
29
|
"Pull work assignment join integration|bash evals/integration/test_pull_work_assignment_join.sh"
|
|
28
30
|
"Ensure-session ownership guard integration|bash evals/integration/test_ensure_session_ownership_guard.sh"
|
|
31
|
+
"Verify-hold gate integration|bash evals/integration/test_verify_hold.sh"
|
|
32
|
+
"Takeover protocol integration|bash evals/integration/test_takeover_protocol.sh"
|
|
29
33
|
"Current.json per-actor integration|bash evals/integration/test_current_json_per_actor.sh"
|
|
30
34
|
"Goal Fit hook integration|bash evals/integration/test_goal_fit_hook.sh"
|
|
31
35
|
"Hook category behavior integration|bash evals/integration/test_hook_category_behaviors.sh"
|
|
@@ -77,10 +81,14 @@ LANE_WORKFLOW_CONTRACTS=(
|
|
|
77
81
|
"Workflow sidecar writer integration"
|
|
78
82
|
"Sidecar field preservation integration"
|
|
79
83
|
"Actor identity resolver integration"
|
|
84
|
+
"CI-runtime actor identity integration"
|
|
80
85
|
"Assignment provider local-file integration"
|
|
81
86
|
"Assignment provider github integration"
|
|
87
|
+
"Stop hook release-with-handoff integration"
|
|
82
88
|
"Pull work assignment join integration"
|
|
83
89
|
"Ensure-session ownership guard integration"
|
|
90
|
+
"Verify-hold gate integration"
|
|
91
|
+
"Takeover protocol integration"
|
|
84
92
|
"Current.json per-actor integration"
|
|
85
93
|
)
|
|
86
94
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 5,
|
|
3
|
+
"source": "reconcile-preflight-fixture:disputed-critique-unsuperseded",
|
|
4
|
+
"note": "#356 shape 3: an un-superseded workflow.critique.review claim with a 'fail' verdict derives status 'disputed' — session-local-failed divergence (a disputed critique always blocks reconcile unless a later same-reviewer pass superseded it via metadata.superseded_by).",
|
|
5
|
+
"claims": [
|
|
6
|
+
{
|
|
7
|
+
"id": "c-disputed-critique",
|
|
8
|
+
"claimType": "workflow.critique.review",
|
|
9
|
+
"value": "fail",
|
|
10
|
+
"status": "disputed",
|
|
11
|
+
"subjectId": "reconcile-preflight-fixture/review-1",
|
|
12
|
+
"facet": "flow-agents.workflow",
|
|
13
|
+
"subjectType": "workflow-critique",
|
|
14
|
+
"fieldOrBehavior": "Found a critical security issue in the auth handler.",
|
|
15
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
16
|
+
"updatedAt": "2026-07-04T00:00:00Z",
|
|
17
|
+
"impactLevel": "medium",
|
|
18
|
+
"verificationPolicyId": "policy:workflow.critique.review",
|
|
19
|
+
"metadata": { "origin": "critique", "reviewer": "tool-code-reviewer", "reviewed_at": "2026-07-04T00:00:00Z" }
|
|
20
|
+
}
|
|
21
|
+
],
|
|
22
|
+
"evidence": [],
|
|
23
|
+
"policies": [
|
|
24
|
+
{
|
|
25
|
+
"id": "policy:workflow.critique.review",
|
|
26
|
+
"claimType": "workflow.critique.review",
|
|
27
|
+
"requiredEvidence": [],
|
|
28
|
+
"acceptanceCriteria": ["A verified verification event must support a workflow.critique.review claim."],
|
|
29
|
+
"reviewAuthority": "system",
|
|
30
|
+
"validityRule": { "kind": "manual" },
|
|
31
|
+
"stalenessTriggers": [],
|
|
32
|
+
"conflictRules": [],
|
|
33
|
+
"impactLevel": "medium"
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
"events": [
|
|
37
|
+
{
|
|
38
|
+
"id": "evt-disputed-critique",
|
|
39
|
+
"claimId": "c-disputed-critique",
|
|
40
|
+
"status": "disputed",
|
|
41
|
+
"actor": "flow-agents/workflow-sidecar",
|
|
42
|
+
"method": "validation",
|
|
43
|
+
"evidenceIds": [],
|
|
44
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
45
|
+
"verifiedAt": "2026-07-04T00:00:00Z"
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 5,
|
|
3
|
+
"source": "reconcile-preflight-fixture:standalone-disputed-session-local",
|
|
4
|
+
"note": "#356 shape 4 (the #292/#384 variant): a NON-critique session-local claim (workflow.check.external, human_attestation-backed) with a re-derived status of 'disputed' and no waiver. Distinct from shape 3 (an un-superseded disputed CRITIQUE) only in claimType/subject — both land on the same session-local-failed divergence, proving a disjoint pre-existing failure can never be smuggled in as a standalone claim; the fix hint directs the agent to document it as prose in a waived non-command summary instead.",
|
|
5
|
+
"claims": [
|
|
6
|
+
{
|
|
7
|
+
"id": "c-standalone-disputed",
|
|
8
|
+
"claimType": "workflow.check.external",
|
|
9
|
+
"value": "fail",
|
|
10
|
+
"status": "disputed",
|
|
11
|
+
"subjectId": "reconcile-preflight-fixture/pre-existing-flaky-integration",
|
|
12
|
+
"facet": "flow-agents.workflow",
|
|
13
|
+
"subjectType": "workflow-check",
|
|
14
|
+
"fieldOrBehavior": "pre-existing integration test failure unrelated to this change",
|
|
15
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
16
|
+
"updatedAt": "2026-07-04T00:00:00Z",
|
|
17
|
+
"impactLevel": "high",
|
|
18
|
+
"verificationPolicyId": "policy:workflow.check.external:human_attestation"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"evidence": [
|
|
22
|
+
{
|
|
23
|
+
"id": "ev-standalone-disputed",
|
|
24
|
+
"claimId": "c-standalone-disputed",
|
|
25
|
+
"evidenceType": "human_attestation",
|
|
26
|
+
"method": "attestation",
|
|
27
|
+
"sourceRef": "reconcile-preflight-fixture/evidence.json",
|
|
28
|
+
"excerptOrSummary": "pre-existing integration test failure, unrelated to this change, observed but not fixed",
|
|
29
|
+
"observedAt": "2026-07-04T00:00:00Z",
|
|
30
|
+
"collectedBy": "flow-agents/workflow-sidecar",
|
|
31
|
+
"passing": false
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"policies": [
|
|
35
|
+
{
|
|
36
|
+
"id": "policy:workflow.check.external:human_attestation",
|
|
37
|
+
"claimType": "workflow.check.external",
|
|
38
|
+
"requiredEvidence": ["human_attestation"],
|
|
39
|
+
"acceptanceCriteria": ["A verified verification event must support a workflow.check.external claim."],
|
|
40
|
+
"reviewAuthority": "system",
|
|
41
|
+
"validityRule": { "kind": "manual" },
|
|
42
|
+
"stalenessTriggers": [],
|
|
43
|
+
"conflictRules": [],
|
|
44
|
+
"impactLevel": "high"
|
|
45
|
+
}
|
|
46
|
+
],
|
|
47
|
+
"events": [
|
|
48
|
+
{
|
|
49
|
+
"id": "evt-standalone-disputed",
|
|
50
|
+
"claimId": "c-standalone-disputed",
|
|
51
|
+
"status": "disputed",
|
|
52
|
+
"actor": "flow-agents/workflow-sidecar",
|
|
53
|
+
"method": "attestation",
|
|
54
|
+
"evidenceIds": ["ev-standalone-disputed"],
|
|
55
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
56
|
+
"verifiedAt": "2026-07-04T00:00:00Z"
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
@@ -278,14 +278,22 @@ flow_agents_node "$WRITER" init-plan "$SESSION_DIR2/${SLUG2}--deliver.md" \
|
|
|
278
278
|
--source-request "Test" --summary "Test" \
|
|
279
279
|
--timestamp "2026-06-26T11:01:00Z" >/dev/null 2>&1
|
|
280
280
|
|
|
281
|
+
# #356: the check-json below carries a real "command" so its test_output evidence
|
|
282
|
+
# has a manifest-matchable execution.label (evidence.execution.label) -- otherwise the new
|
|
283
|
+
# reconcile-preflight gate inside publishDelivery() (#356 Wave 3, AC6) correctly refuses to
|
|
284
|
+
# publish a "kind":"build" claim with test_output evidence but no captured command (a
|
|
285
|
+
# genuinely shape-invalid claim CI's own trust-reconcile.js would ALSO reject -- this is not
|
|
286
|
+
# a preflight-specific false positive, see reconcile-shape.js's finding-1 hardening).
|
|
287
|
+
# TRUST_RECONCILE_COMMANDS makes "node --version" resolve via resolveManifest's legacy
|
|
288
|
+
# fallback tier, matching what a real repo configured with that canonical verify would do.
|
|
281
289
|
flow_agents_node "$WRITER" record-evidence "$SESSION_DIR2" \
|
|
282
290
|
--verdict pass \
|
|
283
|
-
--check-json '{"id":"build","kind":"build","status":"pass","summary":"build passed"}' \
|
|
291
|
+
--check-json '{"id":"build","kind":"build","status":"pass","summary":"build passed","command":"node --version"}' \
|
|
284
292
|
--timestamp "2026-06-26T11:02:00Z" >/dev/null 2>&1
|
|
285
293
|
|
|
286
294
|
# advance-state to delivered: this is the other code path that seals the checkpoint
|
|
287
295
|
SEAL_EXIT=0
|
|
288
|
-
flow_agents_node "$WRITER" advance-state "$SESSION_DIR2" \
|
|
296
|
+
TRUST_RECONCILE_COMMANDS="node --version" flow_agents_node "$WRITER" advance-state "$SESSION_DIR2" \
|
|
289
297
|
--status delivered \
|
|
290
298
|
--phase release \
|
|
291
299
|
--summary "Delivered." \
|
|
@@ -298,10 +306,11 @@ else
|
|
|
298
306
|
_fail "advance-state --status delivered exited $SEAL_EXIT (signing must not break the seal)"
|
|
299
307
|
fi
|
|
300
308
|
|
|
301
|
-
|
|
302
|
-
|
|
309
|
+
# #379: publishDelivery writes to the per-session path delivery/<slug>/trust.bundle.
|
|
310
|
+
if [[ -f "$REPO_ROOT2/delivery/$SLUG2/trust.bundle" ]]; then
|
|
311
|
+
_pass "publish-delivery published into the explicit scratch --repo-root ($REPO_ROOT2/delivery/$SLUG2/trust.bundle, #379 per-session), not process.cwd()"
|
|
303
312
|
else
|
|
304
|
-
_fail "publish-delivery did not write to the explicit scratch --repo-root ($REPO_ROOT2/delivery/trust.bundle) — check the --repo-root plumbing in advanceState"
|
|
313
|
+
_fail "publish-delivery did not write to the explicit scratch --repo-root ($REPO_ROOT2/delivery/$SLUG2/trust.bundle) — check the --repo-root plumbing in advanceState"
|
|
305
314
|
fi
|
|
306
315
|
|
|
307
316
|
if [[ -f "$SESSION_DIR2/trust.checkpoint.json" ]]; then
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_ci_actor_identity.sh — CI-runtime actor identity tier (issue #398, extends #287; ADR 0021 §2).
|
|
3
|
+
#
|
|
4
|
+
# Proves a CI-triggered session gets a STABLE actor derived from the CI provider's published
|
|
5
|
+
# run/job identifiers, sitting ABOVE process-ancestry and BELOW an explicit override / native
|
|
6
|
+
# runtime session id in resolveActor()'s chain. The payoff: the #293 verify-hold gate now ENFORCES
|
|
7
|
+
# for CI sessions (stable identity) instead of degrading to advisory — the root fix for the CI
|
|
8
|
+
# false-block #293 had to work around.
|
|
9
|
+
#
|
|
10
|
+
# Part A — resolveActor() resolution (pure): stability + byte-identity, precedence (override /
|
|
11
|
+
# native runtime win), per-provider detection (GitHub, GitLab), conservative generic fallthrough,
|
|
12
|
+
# recognized-provider-missing-id fallthrough, and env-var injection sanitization.
|
|
13
|
+
# Part B — verify-hold integration: a STABLE CI actor BLOCKS a differing assignment-backed holder
|
|
14
|
+
# (ENFORCE — contrast the ancestry/advisory case in test_verify_hold.sh §1d), and a CI actor's
|
|
15
|
+
# OWN claim is recognized as self and PASSES (the reconstruction-seam fix in resolveEnsureSessionActor).
|
|
16
|
+
#
|
|
17
|
+
# Deterministic, no model spend, self-cleaning, no network, no `gh` process anywhere.
|
|
18
|
+
# Usage: bash evals/integration/test_ci_actor_identity.sh
|
|
19
|
+
|
|
20
|
+
set -uo pipefail
|
|
21
|
+
|
|
22
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
23
|
+
source "$ROOT/evals/lib/node.sh"
|
|
24
|
+
|
|
25
|
+
WRITER="workflow-sidecar"
|
|
26
|
+
ACTOR_IDENTITY_HELPER="$ROOT/scripts/hooks/lib/actor-identity.js"
|
|
27
|
+
|
|
28
|
+
TMPDIR_EVAL="$(mktemp -d)"
|
|
29
|
+
trap 'rm -rf "$TMPDIR_EVAL"' EXIT
|
|
30
|
+
ARTIFACT_ROOT="$TMPDIR_EVAL/artifact-root"
|
|
31
|
+
|
|
32
|
+
errors=0
|
|
33
|
+
pass() { echo " [PASS] $1"; }
|
|
34
|
+
fail() { echo " [FAIL] $1"; errors=$((errors + 1)); }
|
|
35
|
+
|
|
36
|
+
json_query() {
|
|
37
|
+
node -e 'const fs=require("fs"); let cur=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); for (const part of process.argv[2].split(".")) cur=part==="length" ? cur.length : (Array.isArray(cur) ? cur[Number(part)] : cur[part]); console.log(cur);' "$1" "$2"
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
# resolve_field <field:actor|source> <env-assignments...> — run resolveActor under a CLEAN env
|
|
41
|
+
# (only the passed assignments + HOME) so no ambient FLOW_AGENTS_ACTOR/CLAUDE_CODE_SESSION_ID from
|
|
42
|
+
# the eval host leaks into the resolution. Prints the requested field.
|
|
43
|
+
resolve_field() {
|
|
44
|
+
local field="$1"; shift
|
|
45
|
+
# env -i gives a CLEAN slate (no ambient FLOW_AGENTS_ACTOR / CLAUDE_CODE_SESSION_ID / real
|
|
46
|
+
# GITHUB_* from the CI host leaking in); PATH is preserved so `node` resolves.
|
|
47
|
+
env -i HOME="$HOME" PATH="$PATH" "$@" node -e '
|
|
48
|
+
const { resolveActor } = require(process.argv[1]);
|
|
49
|
+
const r = resolveActor(process.env);
|
|
50
|
+
process.stdout.write(process.argv[2] === "source" ? r.source : r.actor);
|
|
51
|
+
' "$ACTOR_IDENTITY_HELPER" "$field"
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
# verify_hold_under_ci <dir> <KEY=VAL...> — run verify-hold with a CONTROLLED CI identity.
|
|
55
|
+
# flow_agents_node is a bash FUNCTION (from node.sh), so it cannot be launched via `env -i`; instead
|
|
56
|
+
# run in a subshell that unsets every identity source that would out-rank the CI tier (an explicit
|
|
57
|
+
# override or a native runtime session id — including the eval host's own, since this suite itself
|
|
58
|
+
# runs under a coding agent) and exports the provided CI vars. When this suite runs in real GitHub
|
|
59
|
+
# Actions, the exported fake ids override the ambient real ones so the assertions stay deterministic.
|
|
60
|
+
verify_hold_under_ci() {
|
|
61
|
+
local dir="$1"; shift
|
|
62
|
+
(
|
|
63
|
+
unset FLOW_AGENTS_ACTOR CLAUDE_CODE_SESSION_ID CODEX_SESSION_ID OPENCODE_SESSION_ID PI_SESSION_ID CLAUDECODE
|
|
64
|
+
while [[ $# -gt 0 ]]; do export "${1?}"; shift; done
|
|
65
|
+
flow_agents_node "$WRITER" verify-hold "$dir"
|
|
66
|
+
)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
# append_liveness_event / write_assignment_record — the proven fixture writers from
|
|
70
|
+
# test_verify_hold.sh (kept byte-identical in shape so the join fixtures behave the same).
|
|
71
|
+
append_liveness_event() {
|
|
72
|
+
local root="$1" subject="$2" actor="$3" at="$4" ttl="${5:-1800}"
|
|
73
|
+
mkdir -p "$root/liveness"
|
|
74
|
+
node -e '
|
|
75
|
+
const fs = require("fs");
|
|
76
|
+
const evt = { type: "claim", subjectId: process.argv[1], actor: process.argv[2], at: process.argv[3], ttlSeconds: Number(process.argv[4]) };
|
|
77
|
+
fs.appendFileSync(process.argv[5], JSON.stringify(evt) + "\n");
|
|
78
|
+
' "$subject" "$actor" "$at" "$ttl" "$root/liveness/events.jsonl"
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
write_assignment_record() {
|
|
82
|
+
local root="$1" slug="$2" sessionId="$3" actorKey="$4"
|
|
83
|
+
mkdir -p "$root/assignment"
|
|
84
|
+
node -e '
|
|
85
|
+
const fs = require("fs");
|
|
86
|
+
const [dest, slug, sessionId, actorKey] = process.argv.slice(1);
|
|
87
|
+
const now = new Date().toISOString();
|
|
88
|
+
const actor = { runtime: "github-actions", session_id: sessionId, host: "eval-host" };
|
|
89
|
+
const rec = {
|
|
90
|
+
schema_version: "1.0", role: "AssignmentClaimRecord", subject_id: slug,
|
|
91
|
+
actor, actor_key: actorKey, claimed_at: now, ttl_seconds: 1800, branch: "main",
|
|
92
|
+
artifact_dir: slug, status: "claimed",
|
|
93
|
+
audit_trail: [{ at: now, transition: "claim", from_actor: null, to_actor: actor, reason: "claim" }],
|
|
94
|
+
};
|
|
95
|
+
fs.writeFileSync(dest, JSON.stringify(rec, null, 2));
|
|
96
|
+
' "$root/assignment/$slug.json" "$slug" "$sessionId" "$actorKey"
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
# GitHub Actions env used across the file (a single job's stable identifiers).
|
|
100
|
+
GHA=(GITHUB_ACTIONS=true GITHUB_RUN_ID=555 GITHUB_RUN_ATTEMPT=1 GITHUB_JOB=deliver)
|
|
101
|
+
|
|
102
|
+
echo "=== CI actor identity (#398) ==="
|
|
103
|
+
|
|
104
|
+
# ─── Part A: resolveActor() resolution ───────────────────────────────────────────────────────
|
|
105
|
+
echo "--- A1. GitHub Actions -> stable ci-runtime actor, byte-identical across invocations ---"
|
|
106
|
+
GH_SRC="$(resolve_field source "${GHA[@]}")"
|
|
107
|
+
GH_ACTOR_1="$(resolve_field actor "${GHA[@]}")"
|
|
108
|
+
GH_ACTOR_2="$(resolve_field actor "${GHA[@]}")"
|
|
109
|
+
[[ "$GH_SRC" == "ci-runtime:github-actions" ]] && pass "source is ci-runtime:github-actions" || fail "source was not ci-runtime:github-actions: '$GH_SRC'"
|
|
110
|
+
[[ "$GH_ACTOR_1" == github-actions:555-1-deliver:* ]] && pass "actor is github-actions:<run>-<attempt>-<job>:<host> ('$GH_ACTOR_1')" || fail "actor shape unexpected: '$GH_ACTOR_1'"
|
|
111
|
+
[[ "$GH_ACTOR_1" == "$GH_ACTOR_2" ]] && pass "actor is byte-identical across repeated invocations (stable)" || fail "actor differed across invocations: '$GH_ACTOR_1' vs '$GH_ACTOR_2'"
|
|
112
|
+
|
|
113
|
+
echo "--- A2. precedence: explicit override and native runtime session id both WIN over CI ---"
|
|
114
|
+
OVR_SRC="$(resolve_field source "${GHA[@]}" FLOW_AGENTS_ACTOR=alice)"
|
|
115
|
+
[[ "$OVR_SRC" == "explicit-override" ]] && pass "FLOW_AGENTS_ACTOR under CI still resolves explicit-override (CI does not win)" || fail "override did not win under CI: '$OVR_SRC'"
|
|
116
|
+
NATIVE_SRC="$(resolve_field source "${GHA[@]}" CLAUDE_CODE_SESSION_ID=sess-xyz)"
|
|
117
|
+
[[ "$NATIVE_SRC" == runtime-session-id:* ]] && pass "native runtime session id under CI still resolves runtime-session-id (CI does not win)" || fail "native runtime id did not win under CI: '$NATIVE_SRC'"
|
|
118
|
+
|
|
119
|
+
echo "--- A3. GitLab CI -> stable gitlab-ci actor from CI_JOB_ID ---"
|
|
120
|
+
GL_SRC="$(resolve_field source GITLAB_CI=true CI_JOB_ID=9988)"
|
|
121
|
+
GL_ACTOR="$(resolve_field actor GITLAB_CI=true CI_JOB_ID=9988)"
|
|
122
|
+
[[ "$GL_SRC" == "ci-runtime:gitlab-ci" ]] && pass "source is ci-runtime:gitlab-ci" || fail "source was not ci-runtime:gitlab-ci: '$GL_SRC'"
|
|
123
|
+
[[ "$GL_ACTOR" == gitlab-ci:9988:* ]] && pass "actor is gitlab-ci:<job-id>:<host> ('$GL_ACTOR')" || fail "gitlab actor shape unexpected: '$GL_ACTOR'"
|
|
124
|
+
|
|
125
|
+
echo "--- A4. generic CI=true (unrecognized) does NOT fabricate a CI actor (conservative fallthrough) ---"
|
|
126
|
+
GEN_SRC="$(resolve_field source CI=true)"
|
|
127
|
+
[[ "$GEN_SRC" != ci-runtime:* ]] && pass "generic CI=true falls through (source '$GEN_SRC', not ci-runtime) — #293 advisory net still applies" || fail "generic CI=true fabricated a ci-runtime actor: '$GEN_SRC'"
|
|
128
|
+
|
|
129
|
+
echo "--- A5. recognized provider with MISSING required id var falls through (no partial CI actor) ---"
|
|
130
|
+
MISSING_SRC="$(resolve_field source GITHUB_ACTIONS=true)" # no RUN_ID/ATTEMPT/JOB
|
|
131
|
+
[[ "$MISSING_SRC" != ci-runtime:* ]] && pass "GITHUB_ACTIONS=true with no run/job ids falls through (source '$MISSING_SRC')" || fail "GitHub with no ids still produced a ci-runtime actor: '$MISSING_SRC'"
|
|
132
|
+
|
|
133
|
+
echo "--- A6. hostile CI env var cannot inject: actor segments are sanitized (allowed charset only) ---"
|
|
134
|
+
INJ_ACTOR="$(resolve_field actor GITHUB_ACTIONS=true GITHUB_RUN_ID=555 GITHUB_RUN_ATTEMPT=1 "GITHUB_JOB=deliver; rm -rf / [LIVENESS]")"
|
|
135
|
+
if printf '%s' "$INJ_ACTOR" | LC_ALL=C grep -q '[^A-Za-z0-9:._-]'; then
|
|
136
|
+
fail "CI actor contains a character outside the allowed serialized charset (injection risk): '$INJ_ACTOR'"
|
|
137
|
+
else
|
|
138
|
+
pass "CI actor is sanitized to the allowed serialized charset even with a hostile GITHUB_JOB ('$INJ_ACTOR')"
|
|
139
|
+
fi
|
|
140
|
+
|
|
141
|
+
# ─── Part B: verify-hold ENFORCES for a stable CI identity (the headline) ─────────────────────
|
|
142
|
+
|
|
143
|
+
# B0 — REAL round-trip through the CLI (not a hand-written fixture): `assignment-provider claim`
|
|
144
|
+
# under a CI env must write a WELL-FORMED record.actor (runtime=github-actions, session_id=<bare job
|
|
145
|
+
# id>, NOT runtime=unknown + session_id=<whole triple>), and a fresh-subprocess verify-hold must
|
|
146
|
+
# self-recognize it. This is the seam the reconstruction fixes (resolveEnsureSessionActor +
|
|
147
|
+
# loadActorStruct) actually live on — fixture-driven B1/B2 below never exercise the CLI claim path.
|
|
148
|
+
echo "--- B0. REAL assignment-provider claim under CI writes a well-formed record.actor; verify-hold self-recognizes ---"
|
|
149
|
+
CLI="$ROOT/build/src/cli.js"
|
|
150
|
+
RT_ROOT="$TMPDIR_EVAL/rt-artifact-root"
|
|
151
|
+
RT_SLUG="ci-roundtrip"
|
|
152
|
+
(
|
|
153
|
+
unset FLOW_AGENTS_ACTOR CLAUDE_CODE_SESSION_ID CODEX_SESSION_ID OPENCODE_SESSION_ID PI_SESSION_ID CLAUDECODE
|
|
154
|
+
export "${GHA[@]}"
|
|
155
|
+
node "$CLI" assignment-provider claim --provider local-file --artifact-root "$RT_ROOT" \
|
|
156
|
+
--subject-id "$RT_SLUG" --branch main --artifact-dir "$RT_SLUG"
|
|
157
|
+
) >"$TMPDIR_EVAL/rt-claim.out" 2>"$TMPDIR_EVAL/rt-claim.err"
|
|
158
|
+
RT_REC="$RT_ROOT/assignment/$RT_SLUG.json"
|
|
159
|
+
if [[ -f "$RT_REC" ]]; then
|
|
160
|
+
pass "assignment-provider claim under CI wrote a record"
|
|
161
|
+
RT_RUNTIME="$(json_query "$RT_REC" "actor.runtime")"
|
|
162
|
+
RT_SESSION="$(json_query "$RT_REC" "actor.session_id")"
|
|
163
|
+
RT_KEY="$(json_query "$RT_REC" "actor_key")"
|
|
164
|
+
[[ "$RT_RUNTIME" == "github-actions" ]] && pass "record.actor.runtime is github-actions (NOT 'unknown' — F1 reconstruction fix)" || fail "record.actor.runtime was '$RT_RUNTIME', expected github-actions (F1 regressed: loadActorStruct not CI-aware)"
|
|
165
|
+
[[ "$RT_SESSION" == "555-1-deliver" ]] && pass "record.actor.session_id is the bare CI job id 555-1-deliver (NOT the whole triple — F1 fix)" || fail "record.actor.session_id was '$RT_SESSION', expected 555-1-deliver (F1 regressed)"
|
|
166
|
+
[[ "$RT_KEY" == github-actions:555-1-deliver:* ]] && pass "record.actor_key is the canonical CI key ('$RT_KEY')" || fail "record.actor_key was '$RT_KEY', expected github-actions:555-1-deliver:<host>"
|
|
167
|
+
else
|
|
168
|
+
fail "assignment-provider claim under CI wrote NO record: $(cat "$TMPDIR_EVAL/rt-claim.out" "$TMPDIR_EVAL/rt-claim.err")"
|
|
169
|
+
fi
|
|
170
|
+
# Fresh-subprocess verify-hold under the same CI env → the actor written by the real claim is self.
|
|
171
|
+
mkdir -p "$RT_ROOT/$RT_SLUG"
|
|
172
|
+
if verify_hold_under_ci "$RT_ROOT/$RT_SLUG" "${GHA[@]}" >"$TMPDIR_EVAL/rt-vh.out" 2>"$TMPDIR_EVAL/rt-vh.err"; then
|
|
173
|
+
pass "verify-hold self-recognizes the CI actor from a REAL claim across subprocesses (end-to-end seam)"
|
|
174
|
+
else
|
|
175
|
+
fail "verify-hold FALSE-BLOCKED the CI actor's own real claim across subprocesses: $(cat "$TMPDIR_EVAL/rt-vh.out" "$TMPDIR_EVAL/rt-vh.err")"
|
|
176
|
+
fi
|
|
177
|
+
[[ "$(json_query "$TMPDIR_EVAL/rt-vh.out" "reason")" == "self_is_holder" ]] && pass "real-round-trip verify-hold reason is self_is_holder" || fail "real-round-trip verify-hold reason was not self_is_holder: $(cat "$TMPDIR_EVAL/rt-vh.out")"
|
|
178
|
+
|
|
179
|
+
echo "--- B1. a stable CI actor BLOCKS a differing, assignment-backed holder (ENFORCE, not advisory) ---"
|
|
180
|
+
BLOCK_SLUG="ci-verify-hold-block"
|
|
181
|
+
BLOCK_DIR="$ARTIFACT_ROOT/$BLOCK_SLUG"
|
|
182
|
+
mkdir -p "$BLOCK_DIR"
|
|
183
|
+
# Held (fresh) by a clearly different actor -> a durable, assignment-backed conflict.
|
|
184
|
+
write_assignment_record "$ARTIFACT_ROOT" "$BLOCK_SLUG" "other-session" "eval-actor-ci-other-holder"
|
|
185
|
+
append_liveness_event "$ARTIFACT_ROOT" "$BLOCK_SLUG" "eval-actor-ci-other-holder" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" 1800
|
|
186
|
+
# Current actor: a STABLE CI identity (GitHub env), NO FLOW_AGENTS_ACTOR — before #398 this would
|
|
187
|
+
# have fallen to ancestry (unstable) and the gate would have degraded to advisory/PASS.
|
|
188
|
+
if verify_hold_under_ci "$BLOCK_DIR" "${GHA[@]}" >"$TMPDIR_EVAL/ci-block.out" 2>"$TMPDIR_EVAL/ci-block.err"; then
|
|
189
|
+
fail "verify-hold under a stable CI identity should BLOCK a differing assignment-backed holder (ENFORCE) — got exit 0: $(cat "$TMPDIR_EVAL/ci-block.out" "$TMPDIR_EVAL/ci-block.err")"
|
|
190
|
+
else
|
|
191
|
+
pass "verify-hold under a stable CI identity BLOCKS a differing assignment-backed holder (the #398 enforce payoff)"
|
|
192
|
+
fi
|
|
193
|
+
[[ "$(json_query "$TMPDIR_EVAL/ci-block.out" "ok")" == "false" ]] && pass "CI-block verify-hold JSON reports ok:false" || fail "CI-block verify-hold JSON did not report ok:false: $(cat "$TMPDIR_EVAL/ci-block.out")"
|
|
194
|
+
CI_BLOCK_REASON="$(json_query "$TMPDIR_EVAL/ci-block.out" "reason")"
|
|
195
|
+
[[ "$CI_BLOCK_REASON" != "actor-identity-unstable-advisory-only" ]] && pass "CI-block reason is NOT the unstable-advisory degradation ('$CI_BLOCK_REASON') — the gate genuinely enforces" || fail "CI actor still degraded to advisory (reason=$CI_BLOCK_REASON) — #398 did not make the CI identity stable"
|
|
196
|
+
|
|
197
|
+
echo "--- B2. a CI actor's OWN claim is recognized as self and PASSES (the reconstruction-seam fix) ---"
|
|
198
|
+
SELF_SLUG="ci-verify-hold-self"
|
|
199
|
+
SELF_DIR="$ARTIFACT_ROOT/$SELF_SLUG"
|
|
200
|
+
mkdir -p "$SELF_DIR"
|
|
201
|
+
# The holder's actor_key IS the CI actor resolveActor() produces under the GitHub env — so the
|
|
202
|
+
# claim written in one CI step is recognized as self at publish in a later step. If the
|
|
203
|
+
# resolveEnsureSessionActor reconstruction diverged (rebuilt an ancestry struct), this would
|
|
204
|
+
# FALSE-BLOCK — exactly the bug #398 removes.
|
|
205
|
+
write_assignment_record "$ARTIFACT_ROOT" "$SELF_SLUG" "self-session" "$GH_ACTOR_1"
|
|
206
|
+
if verify_hold_under_ci "$SELF_DIR" "${GHA[@]}" >"$TMPDIR_EVAL/ci-self.out" 2>"$TMPDIR_EVAL/ci-self.err"; then
|
|
207
|
+
pass "verify-hold PASSES when the CI actor is the holder (self recognized across subprocesses — seam fix)"
|
|
208
|
+
else
|
|
209
|
+
fail "verify-hold FALSE-BLOCKED a CI actor on its OWN claim (the reconstruction seam regressed): $(cat "$TMPDIR_EVAL/ci-self.out" "$TMPDIR_EVAL/ci-self.err")"
|
|
210
|
+
fi
|
|
211
|
+
[[ "$(json_query "$TMPDIR_EVAL/ci-self.out" "ok")" == "true" ]] && pass "CI-self verify-hold JSON reports ok:true" || fail "CI-self verify-hold JSON did not report ok:true: $(cat "$TMPDIR_EVAL/ci-self.out")"
|
|
212
|
+
[[ "$(json_query "$TMPDIR_EVAL/ci-self.out" "reason")" == "self_is_holder" ]] && pass "CI-self verify-hold reason is self_is_holder (the CI actor matched the stored actor_key)" || fail "CI-self verify-hold reason was not self_is_holder: $(cat "$TMPDIR_EVAL/ci-self.out")"
|
|
213
|
+
|
|
214
|
+
echo ""
|
|
215
|
+
if [[ "$errors" -eq 0 ]]; then
|
|
216
|
+
echo "ALL CI ACTOR IDENTITY CHECKS PASSED"
|
|
217
|
+
exit 0
|
|
218
|
+
else
|
|
219
|
+
echo "$errors CHECK(S) FAILED"
|
|
220
|
+
exit 1
|
|
221
|
+
fi
|
|
@@ -21,9 +21,9 @@ json_query() {
|
|
|
21
21
|
node -e 'const fs=require("fs"); let cur=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); for (const part of process.argv[2].split(".")) cur=Array.isArray(cur) ? cur[Number(part)] : cur[part]; console.log(cur);' "$1" "$2"
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
[[ "$(json_query "$TMPDIR_EVAL/audit.json" "totals.scanned")" == "
|
|
24
|
+
[[ "$(json_query "$TMPDIR_EVAL/audit.json" "totals.scanned")" == "16" ]] && pass "audit scans all fixture groups" || fail "audit scans all fixture groups"
|
|
25
25
|
[[ "$(json_query "$TMPDIR_EVAL/audit.json" "totals.retire_candidates")" == "0" ]] && pass "audit finds no unowned retire candidates" || fail "audit finds no unowned retire candidates"
|
|
26
|
-
[[ "$(json_query "$TMPDIR_EVAL/audit.json" "totals.kept")" == "
|
|
26
|
+
[[ "$(json_query "$TMPDIR_EVAL/audit.json" "totals.kept")" == "16" ]] && pass "audit keeps all owned fixture groups" || fail "audit keeps all owned fixture groups"
|
|
27
27
|
|
|
28
28
|
node - "$TMPDIR_EVAL/audit.json" <<'NODE'
|
|
29
29
|
const fs = require("node:fs");
|