agents-relay 2.0.201 → 2.0.203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +7 -6
- package/dist/agent-prompt-builder.js +8 -7
- package/dist/z-adapter.js +10 -29
- package/package.json +1 -1
- package/skills/agents-relay/agents/implementation.agent.md +7 -0
- package/skills/agents-relay/agents/planner.agent.md +8 -1
- package/skills/agents-relay/agents/reviewer.agent.md +6 -0
- package/skills/agents-relay/agents/troubleshooter.agent.md +6 -0
- package/skills/agents-relay/evals/README.md +4 -1
- package/skills/agents-relay/evals/capabilities.json +102 -0
- package/skills/agents-relay/evals/run-codex.mjs +6 -2
package/dist/adapters.js
CHANGED
|
@@ -243,8 +243,9 @@ function declaredPromptReferences(prompt) {
|
|
|
243
243
|
if (start < 0)
|
|
244
244
|
return [];
|
|
245
245
|
const manifest = trustedPrefix.slice(start + marker.length).trim();
|
|
246
|
-
return [...manifest.matchAll(/^- (instructions|context): (
|
|
247
|
-
.map(match => ({ role: match[1], path: match[2], sha256: match[3] }))
|
|
246
|
+
return [...manifest.matchAll(/^- (instructions|context): (\/[^\n]+?)(?: \(sha256:([a-f0-9]{64})\))?$/gm)]
|
|
247
|
+
.map(match => ({ role: match[1], path: match[2], sha256: match[3] ?? basename(match[2], '.md') }))
|
|
248
|
+
.filter(ref => /^[a-f0-9]{64}$/.test(ref.sha256));
|
|
248
249
|
}
|
|
249
250
|
async function accessiblePromptReferences(prompt) {
|
|
250
251
|
const declared = declaredPromptReferences(prompt);
|
|
@@ -262,7 +263,7 @@ async function accessiblePromptReferences(prompt) {
|
|
|
262
263
|
await access(canonical);
|
|
263
264
|
const content = await readFile(canonical, 'utf8');
|
|
264
265
|
if (createHash('sha256').update(content).digest('hex') === reference.sha256)
|
|
265
|
-
readable.push({ ...reference, path: canonical, content });
|
|
266
|
+
readable.push({ ...reference, path: canonical, declaredPath: reference.path, content });
|
|
266
267
|
}
|
|
267
268
|
catch { /* untrusted or missing references are not authorized */ }
|
|
268
269
|
}
|
|
@@ -379,13 +380,13 @@ export class ChatGptAdapter {
|
|
|
379
380
|
let prompt = task.chatgptThreadDelivery?.repository && task.chatgptThreadDelivery.executionId && !graphPrompt
|
|
380
381
|
? taskReferencePrompt(task.chatgptThreadDelivery.repository, task.chatgptThreadDelivery.jobId, task.id, task.chatgptThreadDelivery.executionId, task.input) : task.input;
|
|
381
382
|
const referencePrompt = task.chatgptThreadDelivery && references.length
|
|
382
|
-
? `\n\n[Inline immutable local reference snapshots]\nThese complete verified snapshots are included in this message because this thread delivery does not attach local files. Follow their contents according to their declared roles.\n${references.map(reference => `\n--- ${reference.role}: ${reference.path}
|
|
383
|
+
? `\n\n[Inline immutable local reference snapshots]\nThese complete verified snapshots are included in this message because this thread delivery does not attach local files. Follow their contents according to their declared roles.\n${references.map(reference => `\n--- ${reference.role}: ${reference.path} ---\n${reference.content}\n--- end snapshot ---`).join('\n')}`
|
|
383
384
|
: references.length
|
|
384
|
-
? `\n\n[Attached immutable Agent Contract]\nAfter publishing the required task.started event, read the attached immutable contract snapshot and follow it. Relay owns snapshot integrity; do not hash or compare the original .agent.md. If the attached snapshot is unreadable, report a concrete failure.\n${references.map(reference => `- ${reference.path}
|
|
385
|
+
? `\n\n[Attached immutable Agent Contract]\nAfter publishing the required task.started event, read the attached immutable contract snapshot and follow it. Relay owns snapshot integrity; do not hash or compare the original .agent.md. If the attached snapshot is unreadable, report a concrete failure.\n${references.map(reference => `- ${reference.path}`).join('\n')}`
|
|
385
386
|
: '';
|
|
386
387
|
if (task.chatgptThreadDelivery && declaredPromptReferences(task.input).length) {
|
|
387
388
|
const unresolved = declaredPromptReferences(task.input)
|
|
388
|
-
.map(item => item.path).filter(path => !references.some(reference => reference.
|
|
389
|
+
.map(item => item.path).filter(path => !references.some(reference => reference.declaredPath === path));
|
|
389
390
|
if (unresolved.length)
|
|
390
391
|
throw new Error(`Agent Contract snapshot unavailable or corrupt: ${unresolved.join(', ')}`);
|
|
391
392
|
}
|
|
@@ -27,7 +27,7 @@ export async function promptFileReference(path, role) {
|
|
|
27
27
|
return { path: canonical, role, sha256: createHash('sha256').update(bytes).digest('hex') };
|
|
28
28
|
}
|
|
29
29
|
export function renderAgentPrompt(bundle, adapter) {
|
|
30
|
-
const references = bundle.references.map(ref => `- ${ref.role}: ${ref.path}
|
|
30
|
+
const references = bundle.references.map(ref => `- ${ref.role}: ${ref.path}`).join('\n');
|
|
31
31
|
const header = `[Relay agent: ${bundle.role}; adapter: ${adapter}]\n`;
|
|
32
32
|
const manifest = references ? `\n[Versioned local references]\n${references}\n` : '';
|
|
33
33
|
// ChatGPT has a single user-message surface. The labels are logical, not privileged API roles.
|
|
@@ -102,7 +102,7 @@ export async function buildRelayWorkPrompt(input) {
|
|
|
102
102
|
const bootstrap = input.adapter === 'codex'
|
|
103
103
|
? [
|
|
104
104
|
`Use Codex CLI with project cwd: ${input.projectPath}. Codex -C loads directory AGENTS.md instructions.`,
|
|
105
|
-
...(globalReference ? [`Read global Neo rules once if they are not already loaded: ${globalReference.path}
|
|
105
|
+
...(globalReference ? [`Read global Neo rules once if they are not already loaded: ${globalReference.path}.`] : []),
|
|
106
106
|
'Use local shell and files for referenced contracts and the Relay CLI. Do not call neo bootstrap.',
|
|
107
107
|
]
|
|
108
108
|
: input.adapter === 'chatgpt'
|
|
@@ -115,10 +115,11 @@ export async function buildRelayWorkPrompt(input) {
|
|
|
115
115
|
'Use the same Z interactive bootstrap entry: https://neoy.qili2.com/files/z-chat/bootstrap.md.',
|
|
116
116
|
'Run Neo MCP through authenticated HTTPS. The Z cloud filesystem is not Neo Master. Continue the assigned Task after bootstrap; do not wait for another user message.',
|
|
117
117
|
];
|
|
118
|
-
|
|
118
|
+
// Validate that the selected Relay Skill exists, but let the harness resolve its path.
|
|
119
|
+
await promptFileReference(input.skillPath, 'context');
|
|
119
120
|
const lines = [
|
|
120
121
|
'# Role',
|
|
121
|
-
`You are ${input.agentName} with responsibility at ${reference.path}
|
|
122
|
+
`You are ${input.agentName} with responsibility at ${reference.path}.`,
|
|
122
123
|
'', '## Bootstrap',
|
|
123
124
|
...bootstrap,
|
|
124
125
|
'', '## Contracts',
|
|
@@ -126,7 +127,7 @@ export async function buildRelayWorkPrompt(input) {
|
|
|
126
127
|
? ['MANDATORY: Read the listed Plan/Task contract before planning. A Plan does not publish Task events.']
|
|
127
128
|
: ['MANDATORY: Read every listed Task and Events contract before execution.']),
|
|
128
129
|
'Read independently, in order. Later, more specific contracts take precedence over general rules.',
|
|
129
|
-
...contracts.map((ref, i) => `${i + 1}. ${ref.path}
|
|
130
|
+
...contracts.map((ref, i) => `${i + 1}. ${ref.path}`),
|
|
130
131
|
...(input.agentName === 'planner' ? [
|
|
131
132
|
'Planner lifecycle: use the execution-bound `plan context` from the matching Relay CLI described by the Skill. Run the returned same-version `apply_cli`, and return the exact PLAN_APPLIED receipt. Do not emit Task events for a Plan.',
|
|
132
133
|
] : [
|
|
@@ -137,9 +138,9 @@ export async function buildRelayWorkPrompt(input) {
|
|
|
137
138
|
'Every event MUST use the exact source above. Do not publish `task.blocked` or `task.unblocked`; those are Relay-owned states. Relay must accept the correlated event.',
|
|
138
139
|
]),
|
|
139
140
|
'', '## Contexts',
|
|
140
|
-
...(contexts.length ? ['Read only these additional role-relevant rules. Bootstrap and inherited Agent rules are not duplicated here.', ...contexts.map((ref, i) => `${i + 1}. [${orderedContext[i].source}] ${ref.path}
|
|
141
|
+
...(contexts.length ? ['Read only these additional role-relevant rules. Bootstrap and inherited Agent rules are not duplicated here.', ...contexts.map((ref, i) => `${i + 1}. [${orderedContext[i].source}] ${ref.path}`)] : ['None required at startup. Read files only when selected by the Agent contract or the current Task.']),
|
|
141
142
|
'', '## Task',
|
|
142
|
-
|
|
143
|
+
'Use the agents-relay skill to read authoritative Job and Task/Plan details via the Relay CLI.',
|
|
143
144
|
'All Relay state reads and operations use that version-matched CLI. Publish only required Task events through the separately listed Neo events-bus protocol; never inspect Relay backing storage or HTTP APIs.',
|
|
144
145
|
`Job: ${input.jobId}`,
|
|
145
146
|
`Task: ${input.taskId}`,
|
package/dist/z-adapter.js
CHANGED
|
@@ -31,35 +31,16 @@ export function zTaskBrief(prompt) {
|
|
|
31
31
|
'\n[Authoritative Task input]\n' + local +
|
|
32
32
|
'\n[Output contract]\n' + output;
|
|
33
33
|
}
|
|
34
|
-
/**
|
|
34
|
+
/** Insert only the short-lived credential into the Graph Action's single Bootstrap section. */
|
|
35
35
|
export function zBootstrap(accessToken, prompt) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
'You run in Z cloud Bash. Neo Master is a SEPARATE Mac accessed only by HTTPS Neo MCP at https://neoy.qili2.com/mcp. A Z local file is NOT a Neo Master output.',
|
|
45
|
-
'One-hour OAuth worker credential: ' + accessToken,
|
|
46
|
-
'NEVER print the credential, put it in output, or store it in a repository. Each Bash tool call has a fresh shell. Set the token inside EVERY Bash invocation.',
|
|
47
|
-
'FIRST ACTION: Before any reasoning/planning/bootstrap, make ONE real Bash tool call to send task.started. Relay will timeout without it. Use the exact shell commands:',
|
|
48
|
-
"NEO_ACCESS_TOKEN='" + accessToken + "'",
|
|
49
|
-
"curl -sS --max-time 20 https://neoy.qili2.com/mcp -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' -H \"Authorization: Bearer $NEO_ACCESS_TOKEN\" --data-binary '" + makeEvent('task.started', 'Z worker started') + "'",
|
|
50
|
-
'Check JSON-RPC result (not only HTTP status). Retry once on transient error. DO NOT merely explain this call.',
|
|
51
|
-
'Then follow the manual bootstrap at https://neoy.qili2.com/files/z-chat/bootstrap.md. Via Neo MCP shell exec run "neo bootstrap". Read returned Global rules and only the pinned Event/Task contracts from the Graph Action Prompt.',
|
|
52
|
-
...(instructionSnapshot ? ['After task.started, use Neo MCP fs to read the Relay-pinned agent instruction snapshot: ' + instructionSnapshot + '. Follow it in addition to Neo bootstrap, but keep Task intent from the durable Task brief.'] : []),
|
|
53
|
-
'Work on Neo Master using its MCP fs/shell tools; first discover their schemas with tools/list. A call uses JSON-RPC {"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"shell","arguments":{"command":"help"}}}. Set NEO_ACCESS_TOKEN again inside the SAME Bash tool call as curl.',
|
|
54
|
-
'Before success, validate EVERY explicit acceptance criterion from the Task brief. If the Task requires an exact marker/text, verify that exact literal with Neo fs read or shell grep -F; a nonempty file alone is insufficient. Never publish task.completed when required content is missing.',
|
|
55
|
-
'Verify task output by independent Neo fs read or shell check. LAST ACTION: publish exactly one task.completed after verified output, or task.failed with concrete reason, via the SAME Neo events MCP tool and the SAME source:',
|
|
56
|
-
source,
|
|
57
|
-
'Completion JSON-RPC body: ' + makeEvent('task.completed', 'Completed durable output; verified on Neo Master'),
|
|
58
|
-
'For errors replace task.completed with task.failed and supply the actual error. Do not synthesize Relay state or claim success without an event receipt.',
|
|
59
|
-
'# Retrieve authoritative Job/Task intent through the pinned Relay CLI. Do not use GitHub or PR comments as Task state.',
|
|
60
|
-
'# Graph Action Task Prompt follows unchanged. Its Task identity is authoritative; retrieve Task details using the version-matched Relay CLI.',
|
|
61
|
-
'# Follow the durable Task brief below.',
|
|
62
|
-
].join('\n');
|
|
36
|
+
zExecutionSource(prompt); // Reject an unbound worker before sending credentials.
|
|
37
|
+
if (!accessToken.trim() || /[\r\n]/.test(accessToken))
|
|
38
|
+
throw new Error('Invalid Z worker credential');
|
|
39
|
+
const credential = `Neo OAuth access token: ${accessToken}`;
|
|
40
|
+
const marker = '\n## Bootstrap\n';
|
|
41
|
+
return prompt.startsWith('# Role\n') && prompt.includes(marker)
|
|
42
|
+
? prompt.replace(marker, `${marker}${credential}\n`)
|
|
43
|
+
: `${credential}\n\n${prompt}`; // Legacy prompts lack a Bootstrap section.
|
|
63
44
|
}
|
|
64
45
|
function zRunner() {
|
|
65
46
|
return (_definition, request, _route, signal, context) => pooledWebHelmRun({
|
|
@@ -71,7 +52,7 @@ function zRunner() {
|
|
|
71
52
|
const actualFiles = (request.files ?? []).filter(path => !path.includes('/agents-relay/prompt-references/'));
|
|
72
53
|
const attachments = actualFiles.length ? '\n\nReference paths accessible through Neo MCP fs:\n' + actualFiles.join('\n') : '';
|
|
73
54
|
return { action: 'new-thread', config: {
|
|
74
|
-
message: zBootstrap(token,
|
|
55
|
+
message: zBootstrap(token, zTaskBrief(request.prompt)) + attachments,
|
|
75
56
|
timeout_seconds: 90, wait: false,
|
|
76
57
|
} };
|
|
77
58
|
},
|
package/package.json
CHANGED
|
@@ -13,6 +13,13 @@ Job objective and Task input retrieved from the execution-bound `task context` r
|
|
|
13
13
|
existing design before editing, keep the change narrowly scoped, and preserve
|
|
14
14
|
unrelated user work.
|
|
15
15
|
|
|
16
|
+
For spec-oriented delivery, select `speckit-implement` only when the assigned
|
|
17
|
+
Relay Task owns the **entire** Spec Kit feature and its prerequisites are met:
|
|
18
|
+
that Skill executes every entry in `tasks.md` and marks it complete. If the
|
|
19
|
+
assigned Task owns only part of the feature, follow its linked Spec Kit
|
|
20
|
+
requirements and Relay Task contract **without invoking** the whole-feature
|
|
21
|
+
Skill. Never execute or complete another Relay Task.
|
|
22
|
+
|
|
16
23
|
Follow the mandatory worker self-review loop in
|
|
17
24
|
`../contracts/worker-self-review.md`: implement, critically review, fix every
|
|
18
25
|
actionable finding, then review the entire revised result again until the last
|
|
@@ -2,7 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
You are the default objective planner for all Agents Relay Jobs.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Read the Prompt-listed Task/Job Contract before planning. For spec-oriented
|
|
6
|
+
delivery, select `speckit-plan` for design, `speckit-tasks` for decomposition,
|
|
7
|
+
and `speckit-analyze` for cross-artifact checks, only when the relevant project
|
|
8
|
+
artifacts and prerequisites exist in an authorized checkout. Their output
|
|
9
|
+
informs the current Relay Plan;
|
|
10
|
+
only the execution-bound `apply_cli` can create durable Relay Tasks. A merged
|
|
11
|
+
Agent snapshot may live outside the repository: resolve contracts from the
|
|
12
|
+
Prompt, not relative to this file.
|
|
6
13
|
|
|
7
14
|
You own end-to-end objective convergence. Relay invokes you with the durable Job, existing Tasks, previous planner rounds, and resolved project context. Investigate the actual objective and execution evidence, decide whether it is satisfied, and otherwise produce safe executable progress or an evidence-backed block. A Task Tree is a means to that outcome, not the outcome itself.
|
|
8
15
|
|
|
@@ -13,6 +13,12 @@ Review exactly the immutable target/version specified by the Review Task. Do not
|
|
|
13
13
|
|
|
14
14
|
Reconstruct all material acceptance requirements from the durable Job objective and Task, then evaluate each with direct evidence. Check correctness, completeness, provenance, quality, regressions, explicit approvals and externally observable outcomes where applicable. Missing evidence is not PASS. If validation cannot complete, report task.failed. Keep review independent of implementation work.
|
|
15
15
|
|
|
16
|
+
For spec-oriented delivery, select read-only `speckit-analyze` to check
|
|
17
|
+
`spec.md`, `plan.md`, and `tasks.md` consistency when those artifacts exist.
|
|
18
|
+
Use `speckit-checklist` only as requirements-quality guidance; do not create
|
|
19
|
+
or edit checklists in this read-only Reviewer role. A Spec Kit analysis never
|
|
20
|
+
replaces independent implementation and acceptance-evidence review.
|
|
21
|
+
|
|
16
22
|
## Review Decision
|
|
17
23
|
|
|
18
24
|
Return a concrete approved or changes-required verdict tied to the declared immutable target, with attributed findings, severity and remediation steps. Do not alter artifacts, generate approvals on behalf of users, or perform publication, merges or external actions. Relay owns the review lifecycle and Task/Job completion.
|
|
@@ -13,6 +13,12 @@ You are the recovery orchestrator for a FAILED Agents Relay task. Ordinary Tasks
|
|
|
13
13
|
|
|
14
14
|
Your goal is not to explain the failure. Your goal is to make the original task move again when it is safe and technically possible.
|
|
15
15
|
|
|
16
|
+
For a confirmed spec-to-implementation gap, select `speckit-converge` only
|
|
17
|
+
when its Spec Kit prerequisites are met and the assigned Task authorizes
|
|
18
|
+
editing `tasks.md` in a safe checkout. New entries are planning inputs, not
|
|
19
|
+
durable Relay Tasks. For runtime or lifecycle failures, use Relay recovery
|
|
20
|
+
instead of Spec Kit.
|
|
21
|
+
|
|
16
22
|
Read the independently pinned Task/Job Contract from the Relay Prompt. Only when a shared defect requires a P0/self-job, load `contracts/orchestrator-workflows.md` relative to the Prompt-pinned Relay Skill; do not add it to ordinary recovery startup Context.
|
|
17
23
|
|
|
18
24
|
Read `troubleshooter-qa.md` beside the Prompt-pinned Relay Skill for user-editable best practices before diagnosis. Treat its answers as guidance only, below current evidence and authoritative contracts. After a verified recovery, add or refine a concise Q&A entry when it provides reusable knowledge; never include secrets or unverified claims.
|
|
@@ -4,10 +4,11 @@ Real goal: select and use durable Relay Jobs/Tasks only when needed, plan and ve
|
|
|
4
4
|
|
|
5
5
|
## Capability matrix
|
|
6
6
|
|
|
7
|
-
The provider-independent `capabilities.json` contains
|
|
7
|
+
The provider-independent `capabilities.json` contains 35 cases across:
|
|
8
8
|
- direct vs managed routing (including unauthorized job creation);
|
|
9
9
|
- create/create-only/adopt/local-only and durable dependency handoffs;
|
|
10
10
|
- Planner apply, rejected candidate, and unknown/stale execution;
|
|
11
|
+
- role-specific Spec Kit selection, assigned-Task boundaries, read-only review, and non-spec fallbacks;
|
|
11
12
|
- Task event and status authority, correct execution identity;
|
|
12
13
|
- Reviewer completed-negative verdict vs unavailable review evidence;
|
|
13
14
|
- Troubleshooter diagnosis, recovered Task, shared defect/P0 blocker, human escalation and resumption;
|
|
@@ -32,3 +33,5 @@ The source-level `test/agents-relay-skill-evals.test.js` verifies fixture comple
|
|
|
32
33
|
`npm run test:verification:l4` now runs real generated role/adapter Prompt checks **and** model-based dry-run behavioral scenarios against the generated Prompt and immutable referenced Agent snapshot. `npm run test:verification:l4:contracts` is the deterministic contract suite without model calls; `npm run test:verification:l4:live` probes actual browser/MCP bootstrap. See `test/verification/l4/README.md`.
|
|
33
34
|
|
|
34
35
|
No dry-run eval can prove a terminal Worker event was persisted or a GitHub PR merged. True provider E2E tests must use an isolated Job and independently verify correlated events and durable output. Do not let an eval perform real GitHub/Job actions.
|
|
36
|
+
|
|
37
|
+
Role-specific cases load the current agent Markdown alongside the Relay Skill, so the model chooses behavior against actual role contracts. Use `--only spec-planner-select-plan,spec-implementation-bounded,spec-implementation-entire-feature,spec-reviewer-readonly,spec-recovery-gap,nonspec-planner-no-speckit` for focused selection checks.
|
|
@@ -403,6 +403,108 @@
|
|
|
403
403
|
"job create",
|
|
404
404
|
"service restart"
|
|
405
405
|
]
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
"id": "spec-planner-select-plan",
|
|
409
|
+
"capability": "plan",
|
|
410
|
+
"role": "planner",
|
|
411
|
+
"request": "You are the Relay Planner for a Spec-oriented deliverable. The feature is already specified; the target checkout has valid .specify/ and spec.md. The current Plan is active. What Spec Kit sub-Skill must guide planning before applying the execution-bound Relay Plan?",
|
|
412
|
+
"expected": {
|
|
413
|
+
"decision": "plan",
|
|
414
|
+
"action_contains": [
|
|
415
|
+
"speckit-plan"
|
|
416
|
+
]
|
|
417
|
+
},
|
|
418
|
+
"forbidden": [
|
|
419
|
+
"task create",
|
|
420
|
+
"skip plan context"
|
|
421
|
+
]
|
|
422
|
+
},
|
|
423
|
+
{
|
|
424
|
+
"id": "spec-implementation-bounded",
|
|
425
|
+
"capability": "worker",
|
|
426
|
+
"role": "implementation",
|
|
427
|
+
"request": "Your assigned Relay implementation Task implements only the treasure clue response. This is a Spec-oriented delivery with valid .specify/plan.md/tasks.md. Other Relay Tasks own reveal and rewards. The speckit-implement workflow is whole-feature only. Which workflow do you follow, and why should you not invoke speckit-implement for this subset?",
|
|
428
|
+
"expected": {
|
|
429
|
+
"decision": "worker_context",
|
|
430
|
+
"action_contains": [
|
|
431
|
+
"treasure clue response"
|
|
432
|
+
],
|
|
433
|
+
"alternative_decisions": [
|
|
434
|
+
"direct"
|
|
435
|
+
]
|
|
436
|
+
},
|
|
437
|
+
"forbidden": [
|
|
438
|
+
"speckit-implement",
|
|
439
|
+
"execute all tasks.md"
|
|
440
|
+
]
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
"id": "spec-reviewer-readonly",
|
|
444
|
+
"capability": "review",
|
|
445
|
+
"role": "reviewer",
|
|
446
|
+
"request": "You are the immutable-head Relay Reviewer of a Spec-oriented deliverable. The assigned review includes requirements consistency across spec.md, plan.md and tasks.md, all present in the checkout. Which read-only Spec Kit sub-Skill should guide consistency analysis, while preserving independent immutable-head review?",
|
|
447
|
+
"expected": {
|
|
448
|
+
"decision": "review",
|
|
449
|
+
"action_contains": [
|
|
450
|
+
"speckit-analyze"
|
|
451
|
+
]
|
|
452
|
+
},
|
|
453
|
+
"forbidden": [
|
|
454
|
+
"speckit-checklist as editor",
|
|
455
|
+
"edit review target"
|
|
456
|
+
]
|
|
457
|
+
},
|
|
458
|
+
{
|
|
459
|
+
"id": "spec-recovery-gap",
|
|
460
|
+
"capability": "recover",
|
|
461
|
+
"role": "troubleshooter",
|
|
462
|
+
"request": "You are recovering a failed Relay Task. Verified evidence proves the implementation omitted one Spec requirement. In the assigned editable checkout, speckit-tasks and speckit-implement have completed and the Task explicitly authorizes adding convergence entries to tasks.md. Which relevant Spec Kit sub-Skill can guide convergence before Relay recovery?",
|
|
463
|
+
"expected": {
|
|
464
|
+
"decision": "recover",
|
|
465
|
+
"action_contains": [
|
|
466
|
+
"speckit-converge"
|
|
467
|
+
]
|
|
468
|
+
},
|
|
469
|
+
"forbidden": [
|
|
470
|
+
"declare relay task complete from tasks.md",
|
|
471
|
+
"blind retry"
|
|
472
|
+
]
|
|
473
|
+
},
|
|
474
|
+
{
|
|
475
|
+
"id": "nonspec-planner-no-speckit",
|
|
476
|
+
"capability": "plan",
|
|
477
|
+
"role": "planner",
|
|
478
|
+
"request": "You are the Relay Planner for an ordinary operational incident without any .specify/ project, spec.md or spec-oriented deliverable. Plan the next durable investigation Task. Select the authoritative workflow and avoid unnecessary Spec Kit stages.",
|
|
479
|
+
"expected": {
|
|
480
|
+
"decision": "plan",
|
|
481
|
+
"action_contains": [
|
|
482
|
+
"apply_cli"
|
|
483
|
+
]
|
|
484
|
+
},
|
|
485
|
+
"forbidden": [
|
|
486
|
+
"speckit-plan",
|
|
487
|
+
"speckit-tasks"
|
|
488
|
+
]
|
|
489
|
+
},
|
|
490
|
+
{
|
|
491
|
+
"id": "spec-implementation-entire-feature",
|
|
492
|
+
"capability": "worker",
|
|
493
|
+
"role": "implementation",
|
|
494
|
+
"request": "A single assigned Relay implementation Task explicitly owns all work in one Spec Kit feature. The feature has complete spec.md, plan.md, tasks.md and valid .specify prerequisites; no sibling Relay Tasks own any part. Which Spec Kit sub-Skill should execute the entire feature?",
|
|
495
|
+
"expected": {
|
|
496
|
+
"decision": "worker_context",
|
|
497
|
+
"action_contains": [
|
|
498
|
+
"speckit-implement"
|
|
499
|
+
],
|
|
500
|
+
"alternative_decisions": [
|
|
501
|
+
"direct"
|
|
502
|
+
]
|
|
503
|
+
},
|
|
504
|
+
"forbidden": [
|
|
505
|
+
"skip prerequisites",
|
|
506
|
+
"claim completed before validation"
|
|
507
|
+
]
|
|
406
508
|
}
|
|
407
509
|
]
|
|
408
510
|
}
|
|
@@ -51,13 +51,14 @@ function grade(expected,actual){
|
|
|
51
51
|
const action=String(actual?.action??'');
|
|
52
52
|
if(expected.expected.command && !action.toLowerCase().includes(expected.expected.command.toLowerCase()))mismatches.push('missing action '+expected.expected.command);
|
|
53
53
|
for(const flag of expected.expected.flags??[])if(!action.includes(flag))mismatches.push('missing flag '+flag);
|
|
54
|
+
for(const marker of expected.expected.action_contains??[])if(!action.toLowerCase().includes(marker.toLowerCase()))mismatches.push('missing action marker '+marker);
|
|
54
55
|
// "avoid" may explicitly name forbidden choices. Check the recommended action only.
|
|
55
56
|
for(const forbidden of expected.forbidden??[]){
|
|
56
57
|
const lowered=action.toLowerCase(), forbiddenLower=forbidden.toLowerCase();
|
|
57
58
|
const index=lowered.indexOf(forbiddenLower);
|
|
58
59
|
if(index>=0){
|
|
59
60
|
const preceding=lowered.slice(Math.max(0,index-65),index);
|
|
60
|
-
if(!/(?:without|omit|avoid|never|do not|don\x27t|must not|instead of|rather than)\s*(?:using|include|call|set|the|a|an
|
|
61
|
+
if(!/(?:without|omit|avoid|never|do not|don\x27t|must not|instead of|rather than)\s*(?:(?:using|use|include|call|set|invoke|run|load|the|a|an)\s*)?`?\s*$/.test(preceding))
|
|
61
62
|
mismatches.push('forbidden action '+forbidden);
|
|
62
63
|
}
|
|
63
64
|
}
|
|
@@ -91,10 +92,13 @@ try {
|
|
|
91
92
|
'decision must be one of: direct, create, adopt, inspect, plan, task_create, review, recover, escalate, service, worker_context.',
|
|
92
93
|
'Use the actual Relay CLI or event contracts in the skill. Do not guess output IDs.',
|
|
93
94
|
'In action include exact relevant CLI flags, not only the action name; for task events use exact event names.',
|
|
95
|
+
'For cases with role_contract, follow that exact agent role and use stage-appropriate Spec Kit skills only for spec-oriented delivery.',
|
|
94
96
|
'A forbidden concept may be mentioned in avoid, but do not recommend it in action.',
|
|
95
97
|
'Return one result per case with exactly the same id.'],
|
|
96
98
|
skill,
|
|
97
|
-
cases:set.map(x=>({id:x.id,request:x.request
|
|
99
|
+
cases:await Promise.all(set.map(async x=>({id:x.id,request:x.request,
|
|
100
|
+
...(x.role?{role_contract:await readFile(join(here,'../agents',x.role+'.agent.md'),'utf8')}:{}),
|
|
101
|
+
})))
|
|
98
102
|
};
|
|
99
103
|
await invoke(JSON.stringify(input),file);
|
|
100
104
|
const response=parse(await readFile(file,'utf8'));
|