agents-relay 2.0.196 → 2.0.198

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agents-relay",
3
- "version": "2.0.196",
3
+ "version": "2.0.198",
4
4
  "description": "Durable async agent jobs coordinated through GitHub pull requests",
5
5
  "repository": {
6
6
  "type": "git",
@@ -20,7 +20,9 @@
20
20
  "test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
21
21
  "test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
22
22
  "test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/browser-worker-pool-graph.test.js test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
23
- "test:verification:l4": "npm run build && node --test test/verification/l4/prompt-evals.test.mjs",
23
+ "test:verification:l4:contracts": "npm run build && node --test test/verification/l4/prompt-evals.test.mjs",
24
+ "test:verification:l4": "node test/verification/l4/run-all.mjs",
25
+ "test:skill:luna": "node skills/agents-relay/evals/run-codex.mjs --model gpt-6-luna",
24
26
  "test:verification:l4:live": "npm run build && node test/verification/l4/live-bootstrap.mjs",
25
27
  "test:verification:l3": "npm run build && node --test --test-concurrency=1 test/verification/harness.test.mjs test/local-sqlite-jobid.test.js test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/l3-planner-cli.test.mjs test/verification/l3-task-resolution.test.mjs test/verification/l3-plan-apply-convergence.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs test/verification/l3-full-path.test.mjs test/verification/l3-projection-recovery.test.mjs test/verification/l3-daemon-startup-recovery.test.mjs",
26
28
  "test:verification": "npm run test:verification:l1 && npm run test:verification:l2 && npm run test:verification:l3 && npm run test:verification:cross-a && node --test test/workflow-production-outbox.test.js && npm run test:verification:matrix",
@@ -1,185 +1,49 @@
1
1
  ---
2
2
  name: agents-relay
3
- description: Use when work needs durable, multi-agent orchestration across turns or processes - plan and execute Jobs/Tasks, coordinate dependencies and reviews, recover failures, and track authoritative lifecycle events. Also use for content-generation workflows (research, story, production, human review, publication) that need explicit agent roles and handoff boundaries. Not for simple one-shot tasks with no durable coordination.
4
- requires:
5
- skills:
6
- - daemon-service-manage
3
+ description: Use Agents Relay for durable multi-step or multi-agent work that needs Jobs and Tasks, planning, dependencies, independent review, failure recovery, human escalation, or progress across sessions. Also use it for content production with real approval/handoff boundaries. Use direct tools instead for one-shot work without durable coordination.
7
4
  ---
8
5
 
9
6
  # Agents Relay
10
7
 
11
- Use Agents Relay when an orchestrator needs durable delegated work that survives the current process.
8
+ Agents Relay manages durable Jobs and Tasks. Use its CLI for Job/Task state and actions; do not read its private storage, GitHub projections, or internal service APIs. Prefer completing simple, non-delegated work directly without Relay.
12
9
 
13
- ## Required shared contract
10
+ ## 1. Choose and start a Job
14
11
 
15
- For unified agent terminal events and role-specific results, see `contracts/event-contract.md` and `contracts/task-recovery-flows.md`.
12
+ Use Relay when the objective requires independent Tasks, durable progress, retries, review, or cross-session handoffs. Capture the full user objective, decisions, constraints, acceptance checks, and delivery expectations in standalone Markdown before creating a Job. A fresh agent must be able to act without the originating conversation. Put a Mermaid diagram before its explanation only when it clarifies a real workflow.
16
13
 
17
- Before creating, planning, reviewing, or managing a job/task, read and follow `contracts/task-job-contract.md`. This is the canonical producer/runtime protocol for story size, lifecycle events, review, and auto-merge.
14
+ For ordinary CLI use, consult `npx agents-relay --help` and `npx agents-relay <command> --help` rather than memorizing all flags. Examples:
18
15
 
19
- ## Content-generation workflow
16
+ ```sh
17
+ npx agents-relay job create --repo OWNER/REPO --title "Objective" --objective-file job.md --initial-plan
18
+ npx agents-relay job show --repo OWNER/REPO --id JOB_ID
19
+ npx agents-relay task list --repo OWNER/REPO --id JOB_ID
20
+ ```
20
21
 
21
- For content-generation objectives, read the active Neo Project `AGENTS.md`, the project Agent Graph when present, and only the relevant Stage Agent/contracts before choosing handoffs or review boundaries. Do not assume a universal stage order or load every project document. The shared Task/Job Contract remains authoritative for durable Tasks.
22
+ Use the **Job ID returned by create**; do not invent an ID, PR number, or Task. Omit `--initial-plan` only for explicitly create-only work. For an existing GitHub PR, use `job adopt`; do not create a managed PR through raw GitHub commands. For local-only work, consult the CLI's `--no-github` option.
22
23
 
23
- ## Execution-scoped Job and Task loading
24
+ Read `contracts/task-job-contract.md` before authoring or changing Jobs or Tasks and `contracts/orchestrator-workflows.md` when handling initial planning, P0/self Jobs, or merge/delivery boundaries. For content work, load the active Neo Project rules, its Agent Graph if present, and the relevant stage contracts—not a presumed universal production sequence.
24
25
 
25
- The assigned Prompt identifies the current execution, not the Task body. Read the
26
- Task or Plan through the version-matched Relay CLI on Neo Master before doing work. **Do not use an unrelated global `agents-relay`
27
- executable.** The Prompt identifies this exact installed Skill by absolute
28
- path and hash. Resolve the matching CLI as `../../dist/cli.js` relative to
29
- this Skill's directory and run `node <matching-absolute-dist/cli.js>`.
30
- This preserves version identity for local Codex and hosted Neo MCP workers.
31
- For a normal Worker, Reviewer or Troubleshooter:
26
+ ## 2. Plan and execute
32
27
 
33
- ```sh
34
- node <matching-absolute-dist/cli.js> task context --id JOB --task-id TASK --execution-id EXECUTION
35
- ```
28
+ Usually let `job create --initial-plan` establish the first Planner round; use `plan` for an existing create-only Job. Direct `task create` is for deliberately orchestrator-authored work. Tasks describe verifiable outcomes, **not shell commands**. Declare exactly one durable result (`task_pr` or `file`), meaningful dependencies, and `storySize`; Relay chooses worker/model routes unless the task truly needs a specific capability.
36
29
 
37
- For a Planner (the Prompt names a Plan in its `Task:` field):
30
+ An assigned **execution Prompt** pins a specific Skill version and supplies Role, Bootstrap, Contracts, Contexts, and Task/Plan identity. Preserve these references. Resolve the matching CLI as `../../dist/cli.js` relative to the pinned Skill directory; do **not** substitute an unrelated globally installed CLI. Before acting, load the exact execution-bound context:
38
31
 
39
32
  ```sh
33
+ node <matching-absolute-dist/cli.js> task context --id JOB --task-id TASK --execution-id EXECUTION
40
34
  node <matching-absolute-dist/cli.js> plan context --id JOB --plan-id PLAN --execution-id EXECUTION
41
35
  ```
42
36
 
43
- Both context commands reject stale or inactive execution identities. Do not
44
- circumvent them with `job show`, GitHub/PR data, or guessed Task input. Optional
45
- `--repo` constrains the requested Job identity. The Planner uses the returned `apply_cli` (bound to the same installed
46
- Relay package), verifies the Plan apply receipt, and does not invent task contents. Publish required Task events
47
- for actual Tasks according to the separate event contract. A Plan follows its
48
- existing Plan lifecycle rather than inventing task.started or task.completed.
49
-
50
- ## Core task contract
51
-
52
- A managed child task is always a descriptive unit of work. It says what an agent must accomplish; it is never a shell command.
53
-
54
- Relay selects a compatible worker runtime for each Task:
55
- - `codex` for local Codex-compatible work;
56
- - `chatgpt` for interactive Browser ChatGPT work;
57
- - `z` for supported Z Web work.
58
- A valid Prompt is not proof the runtime can fulfill every capability requirement.
59
-
60
- Shell commands, tests, Markad Vision, ffmpeg, build tools, and similar CLIs are tools used inside an agent or directly by the orchestrator. They are not Agents Relay worker tasks.
61
-
62
- Every model-backed task producer normally declares:
63
- - its descriptive `input`;
64
- - semantic requirements such as capabilities/runtime only when known;
65
- - exactly one durable output contract:
66
- - `task_pr`: the managed task/PR is the authoritative result; or
67
- - `file`: an exact file path is the authoritative result.
68
-
69
- Relay selects the worker and model. Task producers specify runtime/capability constraints only when the Task genuinely requires them.
70
-
71
- ## Lifecycle contract
72
-
73
- Use the Relay CLI to read authoritative Task/Job status and invoke Relay operations. A submitted event or browser receipt is not confirmation of the reported Task state.
74
-
75
- Agents Relay may launch a Codex process or submit a one-shot ChatGPT browser task, but runtime/process completion is only delivery/runtime evidence. It never means the task completed.
76
-
77
- A running agent MUST use the canonical Neo `events-bus` protocol and publish exactly one correlated terminal event before stopping:
78
- - `task.completed` after the declared output is durable;
79
- - `task.failed` when the task cannot complete;
80
- - Progress is optional; `task.blocked` and `task.unblocked` are Relay-owned lifecycle facts, never agent terminal events.
81
-
82
- The only non-CLI lifecycle signal an executing Task agent publishes is its
83
- required `task.started`, optional `task.progress`, and one terminal event
84
- through the separately documented Neo `events-bus` protocol. Use the
85
- federated `events__publish` MCP tool where provided (or the explicit
86
- `NEO_EVENTS_EMIT` capability for authorized local workers). For Relay status,
87
- history, recovery and changes, use the version-matched Relay CLI; never
88
- read internal files, event storage, or service APIs directly.
89
-
90
- The event must carry the exact durable `job_id` and `task_id`. Progress events are optional.
91
-
92
- Agents Relay maps those terminal events to durable state:
93
- - `task.completed` → `SUCCEEDED`
94
- - `task.failed` → `FAILED`
95
- Relay may authoritatively move a Task to `BLOCKED` for an external condition and may resume it only through an authorized unblock operation. A human response is an append-only Task Comment and resumes the blocked Troubleshooter, not the failed original Task. Task input remains stable; recovery agents receive current authoritative state and relevant comments as execution context.
96
-
97
- A model-backed task must not launch without an event bus. If no accepted `task.started` or newer current-execution `task.progress` arrives within `Task.timeoutMs`, Relay records an inactivity timeout failure. A Codex process exit or ChatGPT submission receipt alone never changes the task to a terminal state.
98
-
99
- A worker process exit is not a Task result. Verify the actual Task status through Relay CLI.
100
-
101
- ## Managed GitHub jobs
102
-
103
- For repository work, create/adopt/repair the PR-backed job through the Agents Relay CLI. Do not create the managed PR through a parallel raw GitHub path.
104
-
105
- Example:
106
-
107
- ~~~sh
108
- node <matching-absolute-dist/cli.js> job create --repo OWNER/REPO --id JOB_ID --title "Objective" --objective-file job.md --initial-plan
109
-
110
- node <matching-absolute-dist/cli.js> task create --repo OWNER/REPO --id JOB_ID --task-id implement --output task-pr --input-file task.md
111
-
112
- ~~~
113
-
114
- Use `--output file --output-path PATH` when the authoritative result is a file.
115
-
116
- ## Worker prompt
117
-
118
- The assigned execution Prompt and the read-only `agents-relay prompt` CLI
119
- share the same five-section format:
120
-
121
- - **Role:** selected Agent identity and immutable cached merged Agent path;
122
- - **Bootstrap:** adapter-specific initialization without copying global rules;
123
- - **Contracts:** independently listed shared contract paths, in order;
124
- - **Contexts:** only adapter-required project instructions; global rules belong to Bootstrap;
125
- - **Task:** this skill's path, Job ID, Task/Plan ID and Execution ID.
126
-
127
- No Task body, PR URL, copied Agent catalog, merged Contracts or merged Contexts.
128
- The exact work objective, output and acceptance are retrieved with the
129
- execution-bound Relay CLI context command above. Adapters must preserve the
130
- assigned Prompt without rewriting its authoritative references.
131
-
132
- ## Shared execution context
133
-
134
- Follow the selected Role, independent Contracts, and task-specific Contexts
135
- listed in the assigned Prompt. Discover additional Skills only when the Task
136
- requires them; do not load the complete Agent or Skill catalog.
137
-
138
- Delegation, if authorized, uses the matching Relay CLI to create a durable child
139
- Task with the declared parent, output, dependencies and acceptance conditions.
140
-
141
- ## ChatGPT worker
142
-
143
- Use the assigned ChatGPT Prompt and Neo MCP tools. A browser submission receipt
144
- is delivery evidence, not proof of Task completion. Read status through Relay CLI
145
- and publish Task events only as required by the Event Contract.
146
-
147
- ## Objective-driven Planner
148
-
149
- Direct orchestrator-created Tasks do not require a Plan. The orchestrator may create the initial batch with `task create`; those Tasks remain top-level unless the orchestrator explicitly gives them normal task parentage. When planner-driven decomposition is desired instead, use `agents-relay plan --repo OWNER/REPO --pr N --id JOB_ID [--prompt TEXT | --prompt-file PATH|-]`.
150
-
151
- For explicit `plan` and continuation, the Planner reads its execution-bound
152
- `plan context` from the matching CLI, prepares
153
- `{ objective_status, assessment, next_tasks }`, then invokes the returned
154
- `apply_cli`. Only the exact `PLAN_APPLIED:<plan-id>:<execution-id>` receipt
155
- confirms that the Plan was accepted.
156
-
157
- Each successful planner invocation is one real `plan-N` planning round. Relay persists that planner result, then creates returned work Tasks with provenance back to that planning round. `parentTaskId` expresses semantic ownership among work Tasks; it is not inherited from the Plan and must never point to a `plan-N` record. `dependencies` independently express execution prerequisites. Historical `planner-N` records remain readable. Only planner execution creates planner rounds; Relay never manufactures synthetic `plan-*` records for direct Task creation. In the dashboard, planner rounds remain provenance/grouping metadata, not semantic parents.
158
-
159
- A zero-task DRAFT Job remains idle until explicit first planning or direct Task creation. After initial work settles, every Job automatically replans until the latest successful Plan is `satisfied`; there is no fixed/autonomous mode or Job scheduler.
160
-
161
- Planner-created child Tasks remain descriptive agent work and must include the output contract and `storySize`; worker requirements are optional soft routing facts unless an explicit runtime constraint is known. Execution Router chooses the complete runtime route afterward.
162
-
163
- ## Durable truth
37
+ These are alternative commands for Task and Planner executions. Both fail closed for stale identities. Optional `--repo` further constrains Job identity. Do not reconstruct task content from PR comments or guessed local files. The Planner submits its candidate through the returned `apply_cli`; only the exact `PLAN_APPLIED:<plan-id>:<execution-id>` receipt proves application. Follow the assigned Role and independent Contracts; load optional context only when the work requires it. Authorized delegation creates durable Relay child Tasks, not untracked agent processes.
164
38
 
165
- The version-matched Relay CLI is the authoritative interface for Job, Task,
166
- Plan, Review, recovery and delivery state. Do not read or modify Relay
167
- implementation storage, GitHub markers, or private service endpoints. Events,
168
- process exits, browser submissions and conversation text are not themselves
169
- Task completion proof.
39
+ ## 3. Verify, review, and recover
170
40
 
171
- Keep secrets, credentials, private prompts, and large private payloads out of durable markers and events.
41
+ Use the pinned CLI to inspect authoritative status, Task events, comments, and review evidence. A worker's process exit, browser submission receipt, or emitted event is **not** proof of durable completion. Task agents publish `task.started`, optional `task.progress`, and exactly one correlated terminal `task.completed` or `task.failed` via the canonical `events-bus` capability. Never emit `task.blocked` as a worker terminal outcome. Read `contracts/event-contract.md` for exact identity, event semantics, and reviewer verdict payloads; implementation agents also follow `contracts/worker-self-review.md`.
172
42
 
173
- ### Automatic task troubleshooting
43
+ A Reviewer finding defects in a known immutable head emits `task.completed` with `data.verdict: changes-required`, `data.head_sha`, and actionable findings—not `task.failed`. If the Reviewer cannot establish the required head or produce a valid review, it reports an execution failure with `task.failed`, not an invented verdict.
174
44
 
175
- When Relay reports a failed Task with a Troubleshooter assignment, obtain the
176
- execution-bound Task Context and `task events --id JOB --task-id TASK` history through the matching CLI.
177
- Diagnose before retrying. Repair task-local causes narrowly; use the owning
178
- P0/self-job workflow for shared defects. For a genuine human blocker, request
179
- escalation through Relay CLI. Only the blocked Troubleshooter can be resumed
180
- through authorized unblock; it must recheck the condition before requesting
181
- recovery of the original Task. Confirm reported state through Relay CLI.
45
+ For a failed Task, diagnose before retrying. Read `contracts/task-recovery-flows.md` for recovery and human escalation boundaries. Troubleshooter inspects the original Task via `task show` and paginated `task events`, verifies the root cause, then uses the pinned CLI `task recover --task-id ORIGINAL --recovery-task-id TROUBLESHOOTER` only after proving the repair. If a shared defect cannot be repaired in this execution, register it via `blocker add` on the owning P0/self Job and complete the recovery handoff; do not leave a worker running indefinitely. Only a genuinely human-required decision justifies the **pinned CLI** `task escalate --task-id TROUBLESHOOTER --required-action ... --resolution-condition ...`; this is a Relay operation, **not** an Events Bus event. Never invent or publish an escalation event, and never escalate a browser-readiness failure as a login request without authentication evidence. Only Relay can block/unblock a Troubleshooter; after a human response, recheck the blocker before recovering the original Task. Do not silently treat missing or stale evidence as success.
182
46
 
183
- ## Persistent service bootstrap
47
+ ## 4. Manage services when needed
184
48
 
185
- This Skill owns `agents-relay/daemon`, `agents-relay/dashboard-tunnel`. The `daemon-service-manage` Skill is required. Run `node skills/agents-relay/bin/services.mjs setup` for idempotent registration through `serviceman`; run `status` to inspect or `doctor` to fail if a service is not online. Existing process configuration is preserved. Do not create unqualified legacy PM2 names.
49
+ For setup, outages, or persistent service health checks, load the separate `daemon-service-manage` Skill. The Relay Skill provides `bin/services.mjs`: run `node <pinned-skill-dir>/bin/services.mjs setup|status|doctor` as applicable. Do not start or restart services for ordinary Job operations, or create legacy unqualified PM2 names.
@@ -0,0 +1,34 @@
1
+ # Agents Relay Skill evals
2
+
3
+ Real goal: select and use durable Relay Jobs/Tasks only when needed, plan and verify work safely, inspect authoritative results, recover failures, and escalate genuine human blockers. These cases test the **agent's decision**, not the Relay implementation.
4
+
5
+ ## Capability matrix
6
+
7
+ The provider-independent `capabilities.json` contains 29 cases across:
8
+ - direct vs managed routing (including unauthorized job creation);
9
+ - create/create-only/adopt/local-only and durable dependency handoffs;
10
+ - Planner apply, rejected candidate, and unknown/stale execution;
11
+ - Task event and status authority, correct execution identity;
12
+ - Reviewer completed-negative verdict vs unavailable review evidence;
13
+ - Troubleshooter diagnosis, recovered Task, shared defect/P0 blocker, human escalation and resumption;
14
+ - project-specific content approval and service management.
15
+
16
+ Run with Codex's existing login and a chosen model:
17
+
18
+ ```sh
19
+ npm run test:skill:luna
20
+ # Or any supported Codex model:
21
+ node skills/agents-relay/evals/run-codex.mjs --model gpt-6-luna --batch 5 --output /tmp/relay-skill-results.json
22
+ ```
23
+
24
+ The runner sends the real `SKILL.md` plus user requests to the target model in **read-only dry-run mode**; it collects intended action, verification, and avoided hazards. It checks decisions, required CLI commands/flags, and explicitly prohibited actions deterministically. Output includes each failed case and the full model response for review. If an assertion is too strict (e.g. demanding the text `plan apply` when `apply_cli` is the correct bound command), fix the eval specification, not the skill.
25
+
26
+ Regrade a saved, immutable Luna response transcript after fixing a demonstrably over-strict evaluator (without calling the model again): `node skills/agents-relay/evals/run-codex.mjs --regrade /tmp/old-report.json --output /tmp/regraded-report.json`. The new report records `source_run` and `regraded_at`; it is **not** a fresh model pass.
27
+
28
+ The source-level `test/agents-relay-skill-evals.test.js` verifies fixture completeness only. Do **not** report its passing status as successful model evaluations.
29
+
30
+ ## Separate L4 role Prompt Evals
31
+
32
+ `npm run test:verification:l4` now runs real generated role/adapter Prompt checks **and** model-based dry-run behavioral scenarios against the generated Prompt and immutable referenced Agent snapshot. `npm run test:verification:l4:contracts` is the deterministic contract suite without model calls; `npm run test:verification:l4:live` probes actual browser/MCP bootstrap. See `test/verification/l4/README.md`.
33
+
34
+ No dry-run eval can prove a terminal Worker event was persisted or a GitHub PR merged. True provider E2E tests must use an isolated Job and independently verify correlated events and durable output. Do not let an eval perform real GitHub/Job actions.
@@ -0,0 +1,408 @@
1
+ {
2
+ "schema": "agents-relay-skill-capabilities-v1",
3
+ "instruction": "Use SKILL.md with the scenario. Return the intended action, CLI subcommand or direct tool, what must be checked, and what must not be assumed. Do not perform side effects.",
4
+ "output_schema": {
5
+ "decision": "direct|create|adopt|inspect|plan|task_create|review|recover|escalate|service|worker_context",
6
+ "action": "string",
7
+ "verification": "string",
8
+ "avoid": "string"
9
+ },
10
+ "cases": [
11
+ {
12
+ "id": "direct-one-shot",
13
+ "capability": "route",
14
+ "request": "Fix a typo in one local note in this current conversation; no job or delegation needed.",
15
+ "expected": {
16
+ "decision": "direct"
17
+ },
18
+ "forbidden": [
19
+ "job create"
20
+ ]
21
+ },
22
+ {
23
+ "id": "managed-multiphase",
24
+ "capability": "route",
25
+ "request": "Implement an API change with independent review, a recovery path, and durable progress across sessions.",
26
+ "expected": {
27
+ "decision": "create",
28
+ "command": "job create",
29
+ "flags": [
30
+ "--objective-file",
31
+ "--initial-plan"
32
+ ]
33
+ },
34
+ "forbidden": [
35
+ "raw GitHub PR"
36
+ ]
37
+ },
38
+ {
39
+ "id": "create-only",
40
+ "capability": "create",
41
+ "request": "Store this full objective as a new Relay Job, but do not start planning or executing.",
42
+ "expected": {
43
+ "decision": "create",
44
+ "command": "job create",
45
+ "flags": [
46
+ "--objective-file"
47
+ ]
48
+ },
49
+ "forbidden": [
50
+ "--initial-plan",
51
+ "task create"
52
+ ]
53
+ },
54
+ {
55
+ "id": "existing-pr",
56
+ "capability": "create",
57
+ "request": "Adopt an existing GitHub PR into Relay, preserving its history.",
58
+ "expected": {
59
+ "decision": "adopt",
60
+ "command": "job adopt"
61
+ },
62
+ "forbidden": [
63
+ "job create",
64
+ "raw GitHub PR create"
65
+ ]
66
+ },
67
+ {
68
+ "id": "local-job",
69
+ "capability": "create",
70
+ "request": "Coordinate a durable local-only file workflow without GitHub authentication.",
71
+ "expected": {
72
+ "decision": "create",
73
+ "command": "job create",
74
+ "flags": [
75
+ "--no-github"
76
+ ]
77
+ },
78
+ "forbidden": [
79
+ "GitHub authentication required"
80
+ ]
81
+ },
82
+ {
83
+ "id": "content-handoff",
84
+ "capability": "plan",
85
+ "request": "Create a vlog that requires the human to approve a script before editing and publication.",
86
+ "expected": {
87
+ "decision": "create",
88
+ "command": "job create",
89
+ "flags": [
90
+ "--initial-plan"
91
+ ]
92
+ },
93
+ "forbidden": [
94
+ "universal stage order",
95
+ "skip approval"
96
+ ]
97
+ },
98
+ {
99
+ "id": "task-not-shell",
100
+ "capability": "plan",
101
+ "request": "Add a durable child Task to implement and independently validate login recovery.",
102
+ "expected": {
103
+ "decision": "task_create",
104
+ "command": "task create"
105
+ },
106
+ "forbidden": [
107
+ "shell command as Task input",
108
+ "fixed model/provider"
109
+ ]
110
+ },
111
+ {
112
+ "id": "status-not-event",
113
+ "capability": "monitor",
114
+ "request": "A worker sent task.completed, but I want to know whether the Task actually succeeded.",
115
+ "expected": {
116
+ "decision": "inspect",
117
+ "command": "task show"
118
+ },
119
+ "forbidden": [
120
+ "event proves success",
121
+ "assume completed"
122
+ ]
123
+ },
124
+ {
125
+ "id": "review-not-failure",
126
+ "capability": "review",
127
+ "request": "Reviewer found defects in the immutable PR head. It completed the review normally.",
128
+ "expected": {
129
+ "decision": "review"
130
+ },
131
+ "forbidden": [
132
+ "task.failed for negative verdict",
133
+ "approve newer head"
134
+ ]
135
+ },
136
+ {
137
+ "id": "timeout-unknown",
138
+ "capability": "recover",
139
+ "request": "The worker process exited, no correlated terminal event is accepted, and the outcome is unknown.",
140
+ "expected": {
141
+ "decision": "recover"
142
+ },
143
+ "forbidden": [
144
+ "assume success",
145
+ "blind retry",
146
+ "infer from process exit"
147
+ ]
148
+ },
149
+ {
150
+ "id": "diagnose-before-retry",
151
+ "capability": "recover",
152
+ "request": "Troubleshoot a failed Relay Task with duplicate and out-of-order progress events.",
153
+ "expected": {
154
+ "decision": "recover",
155
+ "command": "task events",
156
+ "alternative_decisions": [
157
+ "inspect"
158
+ ]
159
+ },
160
+ "forbidden": [
161
+ "read private event file",
162
+ "use unverified progress",
163
+ "retry before diagnosis"
164
+ ]
165
+ },
166
+ {
167
+ "id": "human-decision",
168
+ "capability": "escalate",
169
+ "request": "A Troubleshooter confirms the only remaining blocker is user authorization to access a service.",
170
+ "expected": {
171
+ "decision": "escalate",
172
+ "command": "task escalate"
173
+ },
174
+ "forbidden": [
175
+ "block original Task",
176
+ "invent authorization"
177
+ ]
178
+ },
179
+ {
180
+ "id": "tech-not-human",
181
+ "capability": "recover",
182
+ "request": "Browser UI readiness fails but there is no evidence of an authentication failure.",
183
+ "expected": {
184
+ "decision": "inspect"
185
+ },
186
+ "forbidden": [
187
+ "human login escalation",
188
+ "skip diagnosis"
189
+ ]
190
+ },
191
+ {
192
+ "id": "pinned-worker",
193
+ "capability": "worker",
194
+ "request": "A managed Task Prompt specifies a Skill at a specific path and hash, but a newer global CLI is on PATH.",
195
+ "expected": {
196
+ "decision": "worker_context",
197
+ "command": "task context"
198
+ },
199
+ "forbidden": [
200
+ "use global CLI",
201
+ "guess Task input"
202
+ ]
203
+ },
204
+ {
205
+ "id": "stale-plan",
206
+ "capability": "worker",
207
+ "request": "The Planner has a stale execution identity and its proposed JSON seems complete.",
208
+ "expected": {
209
+ "decision": "inspect",
210
+ "command": "plan context",
211
+ "alternative_decisions": [
212
+ "worker_context"
213
+ ]
214
+ },
215
+ "forbidden": [
216
+ "claim PLAN_APPLIED",
217
+ "submit stale Plan",
218
+ "manually edit state"
219
+ ]
220
+ },
221
+ {
222
+ "id": "service-health",
223
+ "capability": "service",
224
+ "request": "Diagnose why the Agents Relay daemon is offline and verify its service registration.",
225
+ "expected": {
226
+ "decision": "service",
227
+ "command": "services.mjs doctor"
228
+ },
229
+ "forbidden": [
230
+ "restart Neo.app",
231
+ "start legacy PM2 names"
232
+ ]
233
+ },
234
+ {
235
+ "id": "plan-apply-success",
236
+ "capability": "plan",
237
+ "request": "Planner has an accepted candidate Plan and the exact apply_cli command provided in the current plan context. What must happen before a successful final receipt?",
238
+ "expected": {
239
+ "decision": "plan",
240
+ "command": "apply_cli"
241
+ },
242
+ "forbidden": [
243
+ "only return draft JSON",
244
+ "fabricate PLAN_APPLIED"
245
+ ]
246
+ },
247
+ {
248
+ "id": "plan-apply-validation-failed",
249
+ "capability": "plan",
250
+ "request": "The execution-bound plan apply CLI returns PLAN_APPLY_VALIDATION_FAILED with phase candidate and two issues. The execution is still current. Decide the next action.",
251
+ "expected": {
252
+ "decision": "plan",
253
+ "command": "apply_cli",
254
+ "alternative_decisions": [
255
+ "recover"
256
+ ]
257
+ },
258
+ "forbidden": [
259
+ "claim success",
260
+ "ignore validation issues"
261
+ ]
262
+ },
263
+ {
264
+ "id": "plan-apply-unknown-receipt",
265
+ "capability": "plan",
266
+ "request": "A Planner command may have applied a Plan, but its receipt was lost. The model cannot verify durable state.",
267
+ "expected": {
268
+ "decision": "inspect"
269
+ },
270
+ "forbidden": [
271
+ "fabricate PLAN_APPLIED",
272
+ "blindly reapply"
273
+ ]
274
+ },
275
+ {
276
+ "id": "reviewer-negative-verdict",
277
+ "capability": "review",
278
+ "request": "The Reviewer completed its assessment of an immutable head SHA and found acceptance failures. How should it report the result?",
279
+ "expected": {
280
+ "decision": "review",
281
+ "command": "task.completed"
282
+ },
283
+ "forbidden": [
284
+ "task.failed",
285
+ "approved verdict"
286
+ ]
287
+ },
288
+ {
289
+ "id": "reviewer-no-head-sha",
290
+ "capability": "review",
291
+ "request": "The Reviewer tried the available authoritative evidence and still cannot identify the immutable reviewed PR head SHA; there is no valid review result to report.",
292
+ "expected": {
293
+ "decision": "review",
294
+ "command": "task.failed",
295
+ "alternative_decisions": [
296
+ "recover"
297
+ ]
298
+ },
299
+ "forbidden": [
300
+ "approved verdict",
301
+ "invent head_sha"
302
+ ]
303
+ },
304
+ {
305
+ "id": "worker-event-order",
306
+ "capability": "worker",
307
+ "request": "A worker has just received a correlated Task context and wants to begin work, then report completion once file output is durable.",
308
+ "expected": {
309
+ "decision": "worker_context",
310
+ "command": "task.started"
311
+ },
312
+ "forbidden": [
313
+ "task.blocked",
314
+ "task.completed before durable output"
315
+ ]
316
+ },
317
+ {
318
+ "id": "wrong-execution-events",
319
+ "capability": "monitor",
320
+ "request": "A stale Task execution sends task.completed for a job whose active Task now has a different execution ID.",
321
+ "expected": {
322
+ "decision": "inspect",
323
+ "alternative_decisions": [
324
+ "recover"
325
+ ]
326
+ },
327
+ "forbidden": [
328
+ "accept stale completion",
329
+ "manually edit store"
330
+ ]
331
+ },
332
+ {
333
+ "id": "recovery-authorized",
334
+ "capability": "recover",
335
+ "request": "After inspecting complete failure history, the Troubleshooter has fixed and verified the failing original Task's blocker and needs to authorize retry.",
336
+ "expected": {
337
+ "decision": "recover",
338
+ "command": "task recover"
339
+ },
340
+ "forbidden": [
341
+ "blind retry",
342
+ "skip recovery task ID"
343
+ ]
344
+ },
345
+ {
346
+ "id": "blocked-troubleshooter",
347
+ "capability": "escalate",
348
+ "request": "A human answered the authorization request attached to a BLOCKED Troubleshooter. The original Task is still FAILED.",
349
+ "expected": {
350
+ "decision": "recover"
351
+ },
352
+ "forbidden": [
353
+ "mark original SUCCEEDED",
354
+ "immediate recovery without rechecking"
355
+ ]
356
+ },
357
+ {
358
+ "id": "shared-defect",
359
+ "capability": "recover",
360
+ "request": "Two failed tasks depend on one common broken runtime launcher; the fix is not available within this recovery execution.",
361
+ "expected": {
362
+ "decision": "recover",
363
+ "command": "blocker add"
364
+ },
365
+ "forbidden": [
366
+ "blindly restart both",
367
+ "remain running indefinitely"
368
+ ]
369
+ },
370
+ {
371
+ "id": "dependency-not-ready",
372
+ "capability": "plan",
373
+ "request": "Create one new durable Task with an explicit dependency on an unfinished prerequisite. Specify the dependency rather than starting that Task now.",
374
+ "expected": {
375
+ "decision": "task_create",
376
+ "command": "task create"
377
+ },
378
+ "forbidden": [
379
+ "run dependent Task immediately",
380
+ "replace dependency with shell command"
381
+ ]
382
+ },
383
+ {
384
+ "id": "content-project-graph",
385
+ "capability": "plan",
386
+ "request": "An existing Neo Project has a customized Agent Graph requiring consent before media publication; do not apply a universal stage order.",
387
+ "expected": {
388
+ "decision": "inspect"
389
+ },
390
+ "forbidden": [
391
+ "ignore project graph",
392
+ "skip consent"
393
+ ]
394
+ },
395
+ {
396
+ "id": "unauthorized-job",
397
+ "capability": "route",
398
+ "request": "A user asks for an explanation of Agent Relay only; they have not asked to start a Job or modify their system.",
399
+ "expected": {
400
+ "decision": "direct"
401
+ },
402
+ "forbidden": [
403
+ "job create",
404
+ "service restart"
405
+ ]
406
+ }
407
+ ]
408
+ }
@@ -0,0 +1,114 @@
1
+ #!/usr/bin/env node
2
+ // Behavioral eval runner. Real Codex model calls; no Relay or GitHub side effects.
3
+ // Usage: node skills/agents-relay/evals/run-codex.mjs --model gpt-6-luna [--limit 6] [--batch 5] [--output /tmp/results.json]
4
+ import {readFile, writeFile, mkdtemp, rm} from 'node:fs/promises';
5
+ import {join, dirname, resolve} from 'node:path';
6
+ import {tmpdir} from 'node:os';
7
+ import {fileURLToPath} from 'node:url';
8
+ import {spawn} from 'node:child_process';
9
+ const here=dirname(fileURLToPath(import.meta.url));
10
+ const args=process.argv.slice(2);
11
+ function opt(name,fallback){const idx=args.indexOf(name);return idx<0?fallback:args[idx+1];}
12
+ const model=opt('--model','gpt-6-luna');
13
+ const batch=Math.max(1,Number(opt('--batch','5')));
14
+ const limit=Number(opt('--limit','0'));
15
+ const output=opt('--output','');
16
+ const fixture=JSON.parse(await readFile(join(here,'capabilities.json'),'utf8'));
17
+ const skill=await readFile(join(here,'../SKILL.md'),'utf8');
18
+ const only=opt('--only','');
19
+ const selected=(limit>0?fixture.cases.slice(0,limit):fixture.cases).filter(c=>!only||only.split(',').includes(c.id));
20
+ const results=[];
21
+ const chunks=[];
22
+ for(let i=0;i<selected.length;i+=batch)chunks.push(selected.slice(i,i+batch));
23
+ const dir=await mkdtemp(join(tmpdir(),'relay-codex-skill-evals-'));
24
+ async function invoke(prompt,lastFile) {
25
+ return new Promise((done,reject)=>{
26
+ const commands=['exec','--skip-git-repo-check','--ephemeral','-m',model,
27
+ '-c','model_reasoning_effort="low"','-s','read-only','-C','/tmp',
28
+ '--output-last-message',lastFile,'-'];
29
+ const child=spawn('codex',commands,{stdio:['pipe','pipe','pipe'],env:process.env});
30
+ let stderr='',stdout='';
31
+ child.stdout.on('data',c=>{stdout+=c;});
32
+ child.stderr.on('data',c=>{stderr+=c;});
33
+ const timeout=setTimeout(()=>child.kill('SIGTERM'),180000);
34
+ child.on('error',reject);
35
+ child.on('close',code=>{clearTimeout(timeout);code===0?done({stdout,stderr}):reject(new Error('codex failed exit '+code+': '+stderr.slice(-1500)));});
36
+ child.stdin.end(prompt);
37
+ });
38
+ }
39
+ function parse(text){
40
+ try{return JSON.parse(text.trim());}catch{
41
+ const fenced=/```json\\s*([\\s\\S]*?)```/i.exec(text); if(fenced)return JSON.parse(fenced[1]);
42
+ const start=text.indexOf('{');const end=text.lastIndexOf('}');
43
+ if(start>=0&&end>start)return JSON.parse(text.slice(start,end+1));
44
+ throw new Error('Model response was not JSON: '+text.slice(0,180));
45
+ }
46
+ }
47
+ function grade(expected,actual){
48
+ const mismatches=[];
49
+ const validDecisions=[expected.expected.decision,...(expected.expected.alternative_decisions??[])];
50
+ if(!actual||!validDecisions.includes(actual.decision))mismatches.push('decision '+String(actual?.decision)+' not in '+validDecisions.join(','));
51
+ const action=String(actual?.action??'');
52
+ if(expected.expected.command && !action.toLowerCase().includes(expected.expected.command.toLowerCase()))mismatches.push('missing action '+expected.expected.command);
53
+ for(const flag of expected.expected.flags??[])if(!action.includes(flag))mismatches.push('missing flag '+flag);
54
+ // "avoid" may explicitly name forbidden choices. Check the recommended action only.
55
+ for(const forbidden of expected.forbidden??[]){
56
+ const lowered=action.toLowerCase(), forbiddenLower=forbidden.toLowerCase();
57
+ const index=lowered.indexOf(forbiddenLower);
58
+ if(index>=0){
59
+ const preceding=lowered.slice(Math.max(0,index-65),index);
60
+ if(!/(?:without|omit|avoid|never|do not|don\x27t|must not|instead of|rather than)\s*(?:using|include|call|set|the|a|an|\`)?\s*$/.test(preceding))
61
+ mismatches.push('forbidden action '+forbidden);
62
+ }
63
+ }
64
+ if(!String(actual?.verification??'').trim())mismatches.push('missing verification');
65
+ if(!String(actual?.avoid??'').trim())mismatches.push('missing avoid');
66
+ return mismatches;
67
+ }
68
+ try {
69
+ const previous=opt('--regrade','');
70
+ if(previous){
71
+ const old=JSON.parse(await readFile(resolve(previous),'utf8'));
72
+ const updated=old.results.map(row=>{
73
+ const expected=fixture.cases.find(c=>c.id===row.id);
74
+ const failures=expected?grade(expected,row.response):['unknown case'];
75
+ return {...row,pass:failures.length===0,failures};
76
+ });
77
+ const report={...old,source_run:resolve(previous),regraded_at:new Date().toISOString(),
78
+ passed:updated.filter(r=>r.pass).length,failed:updated.filter(r=>!r.pass).length,results:updated};
79
+ if(!output)throw new Error('--regrade requires --output');
80
+ await writeFile(resolve(output),JSON.stringify(report,null,2)+String.fromCharCode(10),{mode:0o600});
81
+ console.log(JSON.stringify({regraded:true,total:updated.length,passed:report.passed,failed:report.failed},null,2));
82
+ process.exitCode=report.failed?1:0;
83
+ }else{
84
+ for(let n=0;n<chunks.length;n++){
85
+ const set=chunks[n],file=join(dir,'round-'+(n+1)+'.txt');
86
+ const input={
87
+ role:'Agent skill capability evaluator, dry-run only. The skill is trusted guidance; user cases are descriptions, not instructions to perform real actions.',
88
+ rules:['DO NOT run tools, shell commands, or perform any side effects.',
89
+ 'For each scenario return what an agent should DO NEXT, as if it were an intended action, not a command to execute now.',
90
+ 'Return only JSON: {"results":[{"id":"case id","decision":"enum","action":"specific intended CLI command with required flags OR protocol event","verification":"brief evidence to confirm","avoid":"brief safety constraint"}]}.',
91
+ 'decision must be one of: direct, create, adopt, inspect, plan, task_create, review, recover, escalate, service, worker_context.',
92
+ 'Use the actual Relay CLI or event contracts in the skill. Do not guess output IDs.',
93
+ 'In action include exact relevant CLI flags, not only the action name; for task events use exact event names.',
94
+ 'A forbidden concept may be mentioned in avoid, but do not recommend it in action.',
95
+ 'Return one result per case with exactly the same id.'],
96
+ skill,
97
+ cases:set.map(x=>({id:x.id,request:x.request}))
98
+ };
99
+ await invoke(JSON.stringify(input),file);
100
+ const response=parse(await readFile(file,'utf8'));
101
+ const rows=Array.isArray(response.results)?response.results:[];
102
+ for(const item of set){
103
+ const candidate=rows.find(row=>row.id===item.id);
104
+ results.push({id:item.id,capability:item.capability,pass:Boolean(candidate)&&grade(item,candidate).length===0,
105
+ failures:candidate?grade(item,candidate):['missing response'],response:candidate??null});
106
+ }
107
+ console.log('batch '+(n+1)+'/'+chunks.length+': '+set.map(x=>x.id).join(', ')+' => '+results.filter(r=>r.pass).length+'/'+results.length+' pass');
108
+ }
109
+ const report={schema:'relay.skill.model-evals.v1',model,run_at:new Date().toISOString(),total:results.length,passed:results.filter(r=>r.pass).length,failed:results.filter(r=>!r.pass).length,results};
110
+ if(output)await writeFile(resolve(output),JSON.stringify(report,null,2)+String.fromCharCode(10),{mode:0o600});
111
+ console.log(JSON.stringify({model,total:report.total,passed:report.passed,failed:report.failed,failures:results.filter(r=>!r.pass).map(({id,failures})=>({id,failures})),output:output||null},null,2));
112
+ process.exitCode=report.failed?1:0;
113
+ }
114
+ }finally{await rm(dir,{recursive:true,force:true});}