@mjasnikovs/pi-task 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/task/auto-orchestrator.js +70 -1
- package/dist/task/contracts.d.ts +54 -0
- package/dist/task/contracts.js +226 -0
- package/dist/task/gate-deps.js +8 -1
- package/dist/task/phases.d.ts +8 -0
- package/dist/task/phases.js +62 -7
- package/dist/task/prompts.d.ts +4 -4
- package/dist/task/prompts.js +12 -7
- package/dist/task/skip-escape.d.ts +25 -0
- package/dist/task/skip-escape.js +80 -0
- package/dist/task/task-gates.js +17 -5
- package/dist/task/verify-work.d.ts +24 -3
- package/dist/task/verify-work.js +119 -9
- package/dist/task/wiring-claims.d.ts +64 -0
- package/dist/task/wiring-claims.js +149 -0
- package/package.json +1 -1
package/dist/task/prompts.js
CHANGED
|
@@ -61,7 +61,7 @@ ${title}`;
|
|
|
61
61
|
* "Scaffold …" title re-expands the entire design into one task (validated: a real
|
|
62
62
|
* /task-auto run implemented all 24 steps under step 1).
|
|
63
63
|
*/
|
|
64
|
-
const REFINE_PROMPT = (raw, planContext, existingFiles) => `${planContext ? planContext + '\n\n---\n\n' : ''}You receive a user's task description for an AI coding agent. Rewrite it to be unambiguous and actionable.
|
|
64
|
+
const REFINE_PROMPT = (raw, planContext, existingFiles, contracts) => `${planContext ? planContext + '\n\n---\n\n' : ''}You receive a user's task description for an AI coding agent. Rewrite it to be unambiguous and actionable.
|
|
65
65
|
|
|
66
66
|
Output structure (four sections, exact headings, in this order):
|
|
67
67
|
|
|
@@ -86,8 +86,9 @@ Rules:
|
|
|
86
86
|
- Preserve every concrete identifier verbatim (paths, function names, ports, env vars, file:line refs).
|
|
87
87
|
- Do not invent requirements not implied by the input.
|
|
88
88
|
- If the task references a design/spec document (an @-path or a named spec file), READ it and treat it as authoritative. Carry its concrete schema verbatim into GOAL/CONSTRAINTS — table and column names, types, endpoint methods and paths, enum values. The task title is only a pointer into that spec: where the title and the spec disagree, follow the spec, and never introduce a table, column, endpoint, or dependency the spec does not define.
|
|
89
|
+
- CITE interface WIRING, do NOT synthesize it. A wiring specific — how modules/endpoints/files connect (a mount prefix, a route/mount table, a module→path mapping, an exported function/type signature, a file or module layout) — must be citable from the design or the CROSS-SLICE CONTRACTS. The design often pins the interface FACTS (the exact endpoint paths, exported names, layouts) WITHOUT stating the wiring that produces them; when it does, any wiring you write MUST reproduce those pinned facts EXACTLY. Do NOT infer a "uniform" or "tidy" pattern from them — e.g. do not assume one module maps to one mount prefix when the design's pinned facts for that module do not all sit under a single prefix (that exact inference is a seam bug: the consumers follow the pinned facts, the assembly follows your invented pattern, and the seam ships broken). If the design pins neither the fact nor the wiring, leave the detail unspecified rather than inventing a specific.
|
|
89
90
|
- Do not output any preamble, commentary, or markdown headings beyond the four sections above.
|
|
90
|
-
${existingFiles && existingFiles.trim() ? `\n${existingFiles.trim()}\n` : ''}
|
|
91
|
+
${contracts && contracts.trim() ? `\n${contracts.trim()}\n` : ''}${existingFiles && existingFiles.trim() ? `\n${existingFiles.trim()}\n` : ''}
|
|
91
92
|
Task: ${raw}`;
|
|
92
93
|
// ─── Research fan-out prompts ─────────────────────────────────────────────────
|
|
93
94
|
const RESEARCH_READ_ONLY_CONSTRAINT = `IMPORTANT: You are ONLY allowed to READ. Do NOT create, modify, or delete any files. Use the read, grep, find, and ls tools to inspect the repo.`;
|
|
@@ -288,7 +289,7 @@ function composeRetryEmphasis(problem) {
|
|
|
288
289
|
}
|
|
289
290
|
return `\nPREVIOUS ATTEMPT was invalid (${problem}). Ensure all four sections are present and the output starts with the literal word GOAL.\n`;
|
|
290
291
|
}
|
|
291
|
-
const COMPOSE_PROMPT = (refined, research, qa, retryProblem) => `You are composing the final implementation spec for an AI coding agent. Combine the refined task, the research, and the user's Q&A answers into one spec.
|
|
292
|
+
const COMPOSE_PROMPT = (refined, research, qa, retryProblem, contracts) => `You are composing the final implementation spec for an AI coding agent. Combine the refined task, the research, and the user's Q&A answers into one spec.
|
|
292
293
|
|
|
293
294
|
CRITICAL FORMAT RULES (read first):
|
|
294
295
|
- Output the spec as plain markdown text. Do NOT wrap your entire output in a code block, shell fence, or heredoc. Do NOT prefix with \`\`\`sh / \`\`\`bash. Do NOT use \`cat << EOF > file\` patterns. Your response begins literally with "GOAL" on the first line.
|
|
@@ -334,14 +335,15 @@ Research:
|
|
|
334
335
|
${research}
|
|
335
336
|
|
|
336
337
|
User Q&A:
|
|
337
|
-
${qa}
|
|
338
|
+
${qa}
|
|
339
|
+
${contracts && contracts.trim() ? `\n${contracts.trim()}\n` : ''}`;
|
|
338
340
|
// Fast triage pass run before the (expensive) full rewrite. It produces either
|
|
339
341
|
// the single token CLEAN — meaning the compose draft needs no rewrite — or a
|
|
340
342
|
// short defect list. When CLEAN, the orchestrator returns the draft unchanged
|
|
341
343
|
// and skips the rewrite entirely; otherwise the defects are fed into
|
|
342
344
|
// CRITIQUE_PROMPT as a focus list so the rewrite targets real problems instead
|
|
343
345
|
// of re-deriving them from scratch.
|
|
344
|
-
const CRITIQUE_TRIAGE_PROMPT = (spec, refined, qa) => `You are triaging an implementation spec for an AI coding agent. Decide whether it needs a rewrite. Do NOT rewrite it — only judge it.
|
|
346
|
+
const CRITIQUE_TRIAGE_PROMPT = (spec, refined, qa, contracts) => `You are triaging an implementation spec for an AI coding agent. Decide whether it needs a rewrite. Do NOT rewrite it — only judge it.
|
|
345
347
|
|
|
346
348
|
The refined task and the user's Q&A below are GROUND TRUTH. Judge the spec against them. Look for SUBSTANTIVE defects only:
|
|
347
349
|
- ambiguity that would let the agent build the wrong thing
|
|
@@ -349,7 +351,8 @@ The refined task and the user's Q&A below are GROUND TRUTH. Judge the spec again
|
|
|
349
351
|
- a VERIFY block that is missing, unrunnable, full of placeholders, or does not exercise the surface the task touches
|
|
350
352
|
- scope drift: requirements, files, or deliverables not implied by the refined task or Q&A
|
|
351
353
|
- a dropped or weakened CONSTRAINT from the refined task
|
|
352
|
-
|
|
354
|
+
- a synthesized interface WIRING specific — a mount/route table, a module→path mapping, an exported signature, a file layout — that the design does not pin AND that does not reproduce the design's pinned interface facts. A "uniform" pattern (one module → one mount prefix, etc.) applied to an interface whose pinned facts are NOT uniform is a SEAM BUG: flag it naming the pinned fact it contradicts.
|
|
355
|
+
${contracts && contracts.trim() ? `\n${contracts.trim()}\n` : ''}
|
|
353
356
|
Do NOT flag cosmetic wording, style, or anything you would change only to "polish" prose. The bar is: would this defect change what the agent builds or whether the work can be verified?
|
|
354
357
|
|
|
355
358
|
Output format — read carefully:
|
|
@@ -364,7 +367,7 @@ ${qa}
|
|
|
364
367
|
|
|
365
368
|
Spec to triage:
|
|
366
369
|
${spec}`;
|
|
367
|
-
const CRITIQUE_PROMPT = (spec, refined, qa, addVerifyEmphasis, triageDefects = null) => `You are reviewing the implementation spec below for ambiguity, weak acceptance criteria, and missing or unrunnable VERIFY commands.
|
|
370
|
+
const CRITIQUE_PROMPT = (spec, refined, qa, addVerifyEmphasis, triageDefects = null, contracts) => `You are reviewing the implementation spec below for ambiguity, weak acceptance criteria, and missing or unrunnable VERIFY commands.
|
|
368
371
|
|
|
369
372
|
CRITICAL FORMAT RULES (read first):
|
|
370
373
|
- Output the rewritten spec as plain markdown. Do NOT wrap your entire output in a code block, shell fence, or heredoc. Do NOT prefix with \`\`\`sh / \`\`\`bash. Do NOT use \`cat << EOF > file\` patterns. Your response begins literally with "GOAL" on the first line.
|
|
@@ -378,6 +381,7 @@ SCOPE RULES (equally critical — do not break these):
|
|
|
378
381
|
- CONSTRAINTS from the refined task MUST be preserved in spirit. Do not silently drop or weaken them.
|
|
379
382
|
- If the spec below is malformed, empty, or wrapped in a heredoc, reconstruct it from the refined task and Q&A — not from your own invention.
|
|
380
383
|
- Your job is to tighten language, sharpen acceptance criteria, and ensure VERIFY is runnable. Not to redesign the task.
|
|
384
|
+
- WIRING vs pinned facts: if the spec states interface wiring (a mount/route table, a module→path mapping, an exported signature, a file layout), reconcile EACH wiring specific against the design's pinned interface facts (the CROSS-SLICE CONTRACTS below, if present, are those facts quoted verbatim). Keep every wiring specific that reproduces the pinned facts exactly; CORRECT any that do not; and do NOT invent wiring the design leaves unspecified. Watch specifically for a "uniform" pattern (one module → one mount prefix, one naming scheme) applied to an interface whose pinned facts are NOT uniform — that is a seam bug, fix only the entry that breaks, and leave the conforming entries unchanged.
|
|
381
385
|
|
|
382
386
|
Rewrite the spec in the same four-section format (GOAL, CONSTRAINTS, ACCEPTANCE, VERIFY). Fix any issues you find within the scope rules above.
|
|
383
387
|
|
|
@@ -390,6 +394,7 @@ VERIFY QUALITY CHECK (apply during the rewrite):
|
|
|
390
394
|
- Never accept \`true\`, \`echo ok\`, or other no-op commands as VERIFY content.
|
|
391
395
|
|
|
392
396
|
${addVerifyEmphasis ? 'REQUIRED: The output MUST include a VERIFY: section followed by a ```sh fenced block of runnable shell commands. The previous attempt was missing this.' : ''}
|
|
397
|
+
${contracts && contracts.trim() ? `\n${contracts.trim()}\n` : ''}
|
|
393
398
|
${triageDefects ?
|
|
394
399
|
`FOCUS — a triage pass already found these specific defects. Fix every one of them in your rewrite (without breaking the scope rules above):\n${triageDefects}\n`
|
|
395
400
|
: ''}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export interface SkipEscapeFinding {
|
|
2
|
+
/** The offending VERIFY command line, verbatim. */
|
|
3
|
+
line: string;
|
|
4
|
+
/** Why it is a skip-escape (human- and prompt-readable). */
|
|
5
|
+
reason: string;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Scan a composed spec's VERIFY block for skip-announcing escapes. Returns one
|
|
9
|
+
* finding per offending command line; empty when the spec has no VERIFY block or
|
|
10
|
+
* no skip-escapes. Comment/blank lines are already dropped by parseVerifyBlock.
|
|
11
|
+
*/
|
|
12
|
+
export declare function findSkipEscapes(spec: string): SkipEscapeFinding[];
|
|
13
|
+
/**
|
|
14
|
+
* Render skip-escape findings as a defect block for the critique rewrite: a
|
|
15
|
+
* numbered instruction list the rewrite must resolve (remove the escape / run the
|
|
16
|
+
* check unconditionally, or drop the check if it is genuinely not required).
|
|
17
|
+
*/
|
|
18
|
+
export declare function skipEscapeDefectText(findings: SkipEscapeFinding[]): string;
|
|
19
|
+
/**
|
|
20
|
+
* Render skip-escape findings as verify-child prompt lines (the deterministic
|
|
21
|
+
* finding that makes rule 5c fire reliably — the model does not self-discover a
|
|
22
|
+
* graceful skip-escape, but acts on a finding that names the exact line). Empty
|
|
23
|
+
* findings → empty array (caller emits no block).
|
|
24
|
+
*/
|
|
25
|
+
export declare function skipEscapeVerifyFindings(findings: SkipEscapeFinding[]): string[];
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic skip-escape scanner for authored VERIFY blocks (run-8 F2).
|
|
3
|
+
*
|
|
4
|
+
* A skip-escape is a `||` fallback that lets a REQUIRED check pass SILENTLY when
|
|
5
|
+
* its tool is absent or it fails — e.g. `playwright test … || echo "skipping"`.
|
|
6
|
+
* mx5 run-8 shipped a blank/dead app partly because its only behavioral smoke
|
|
7
|
+
* tests were wrapped this way: the tool was absent, the check silently skipped,
|
|
8
|
+
* and the verify child blessed it as "correctly skipped".
|
|
9
|
+
*
|
|
10
|
+
* FP-MEASURED on the historical VERIFY blocks (~/hub/mx5/.pi-tasks, 22 tasks): a
|
|
11
|
+
* blanket `|| true` flag is ~90% false positives — teardown (`kill … || true`,
|
|
12
|
+
* `docker compose down … || true`), setup (`… install … || true`), and negative
|
|
13
|
+
* tests (`… && exit 1 || true`, where `|| true` catches an EXPECTED failure). Of
|
|
14
|
+
* 45 `||` uses, exactly one was the real F2 skip-escape, and it ANNOUNCED the skip
|
|
15
|
+
* ("Playwright not available — skipping browser smoke test"). So the crisp,
|
|
16
|
+
* ~zero-FP signal is a fallback whose text ADMITS it is dodging the check — that is
|
|
17
|
+
* the actionable finding here. Bare `|| true` is left to the verify child's runtime
|
|
18
|
+
* rule 5c, which can actually observe whether the check ran (a static scan cannot
|
|
19
|
+
* tell a required-check `|| true` from a teardown `|| true`).
|
|
20
|
+
*
|
|
21
|
+
* Pure shell-shape / text analysis; no stack or tool-name assumptions.
|
|
22
|
+
*/
|
|
23
|
+
import { parseVerifyBlock } from './spec-validation.js';
|
|
24
|
+
/**
|
|
25
|
+
* A `||` (optionally after a `2>/dev/null`) whose fallback text ADMITS it is
|
|
26
|
+
* skipping / that a tool is unavailable. Anchored on the fallback wording, so it
|
|
27
|
+
* fires on `|| echo "skipping"`, `|| { echo "playwright not installed"; }`,
|
|
28
|
+
* `|| echo "runner unavailable — skipped"`, and misses benign teardown `|| true`.
|
|
29
|
+
*/
|
|
30
|
+
const SKIP_ANNOUNCE_RE = /\|\|[^|]*\b(skip|skipping|skipped|not\s+installed|not\s+available|unavailable)\b/i;
|
|
31
|
+
/**
|
|
32
|
+
* Scan a composed spec's VERIFY block for skip-announcing escapes. Returns one
|
|
33
|
+
* finding per offending command line; empty when the spec has no VERIFY block or
|
|
34
|
+
* no skip-escapes. Comment/blank lines are already dropped by parseVerifyBlock.
|
|
35
|
+
*/
|
|
36
|
+
export function findSkipEscapes(spec) {
|
|
37
|
+
const cmds = parseVerifyBlock(spec);
|
|
38
|
+
if (!cmds)
|
|
39
|
+
return [];
|
|
40
|
+
const found = [];
|
|
41
|
+
for (const { raw } of cmds) {
|
|
42
|
+
if (SKIP_ANNOUNCE_RE.test(raw)) {
|
|
43
|
+
found.push({
|
|
44
|
+
line: raw,
|
|
45
|
+
reason: 'its `||` fallback announces skipping the check when a tool is absent — a '
|
|
46
|
+
+ 'required check must run unconditionally, not self-waive into a silent pass'
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return found;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Render skip-escape findings as a defect block for the critique rewrite: a
|
|
54
|
+
* numbered instruction list the rewrite must resolve (remove the escape / run the
|
|
55
|
+
* check unconditionally, or drop the check if it is genuinely not required).
|
|
56
|
+
*/
|
|
57
|
+
export function skipEscapeDefectText(findings) {
|
|
58
|
+
return [
|
|
59
|
+
'SKIP-ESCAPE in the VERIFY block — a required check is wrapped so that a missing',
|
|
60
|
+
'tool or a failure passes SILENTLY (run-8 F2: the only smoke tests shipped this',
|
|
61
|
+
'way, skipped unnoticed, and a blank app was blessed). Rewrite the VERIFY block so',
|
|
62
|
+
'each of these checks RUNS UNCONDITIONALLY and its failure fails the block — remove',
|
|
63
|
+
'the `|| echo skipping`-style fallback. PREFER a check that needs no special tool at',
|
|
64
|
+
'all (start the artifact and probe its real behavior directly). Do NOT merely reshape',
|
|
65
|
+
'the escape into a `command -v X`/`if`-guard that still skips silently when the tool',
|
|
66
|
+
'is absent — that is the same defect: if the check truly needs a tool, its absence',
|
|
67
|
+
'must make the block EXIT NON-ZERO (surface it), never exit 0. If a check genuinely',
|
|
68
|
+
'cannot be required here, remove it entirely rather than leaving a self-waiving stub:',
|
|
69
|
+
...findings.map((f, i) => ` ${i + 1}. ${f.line}`)
|
|
70
|
+
].join('\n');
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Render skip-escape findings as verify-child prompt lines (the deterministic
|
|
74
|
+
* finding that makes rule 5c fire reliably — the model does not self-discover a
|
|
75
|
+
* graceful skip-escape, but acts on a finding that names the exact line). Empty
|
|
76
|
+
* findings → empty array (caller emits no block).
|
|
77
|
+
*/
|
|
78
|
+
export function skipEscapeVerifyFindings(findings) {
|
|
79
|
+
return findings.map(f => `${f.line} — ${f.reason}`);
|
|
80
|
+
}
|
package/dist/task/task-gates.js
CHANGED
|
@@ -96,17 +96,29 @@ export async function runGatesForTask(ctxIn, deps, p) {
|
|
|
96
96
|
continue;
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
99
|
+
// UNOBSERVED (rule 5c): a spec-required behavioral check could not run because
|
|
100
|
+
// its observation tooling is absent. An unattended AUTOFIX re-run cannot install
|
|
101
|
+
// a missing tool, so it would only burn MAX_AUTO_AUTOFIX turns and re-FAIL — the
|
|
102
|
+
// decision (provision the tool, or accept the unproven behavior) is the human's.
|
|
103
|
+
// Skip the (moot) recommendation research and force the picker.
|
|
104
|
+
const isUnobserved = verified.unobserved === true;
|
|
105
|
+
const recOutcome = isUnobserved ? { recommend: 'autofix', rationale: failReason }
|
|
106
|
+
: deps.recommend ?
|
|
107
|
+
await deps.recommend(active, p.cwd, p.title, p.taskId, failReason)
|
|
108
|
+
: { recommend: 'autofix', rationale: failReason };
|
|
109
|
+
await rec(isUnobserved ?
|
|
110
|
+
'resolution: verify UNOBSERVED — spec-required check could not run (tooling absent); '
|
|
111
|
+
+ 'forcing the human picker, an unattended re-run cannot provision it'
|
|
112
|
+
: `resolution: recommended ${recOutcome.recommend.toUpperCase()}`);
|
|
103
113
|
// AUTO-RESOLVE the AUTOFIX path: when the research says the work is
|
|
104
114
|
// genuinely wrong, re-run the fix WITHOUT prompting the user. The picker is
|
|
105
115
|
// reserved for the ACCEPT recommendation (the human decides whether to bless
|
|
106
116
|
// an artifact the gate FAILed) and for the bounded fallback: after
|
|
107
117
|
// MAX_AUTO_AUTOFIX consecutive unattended attempts that still FAIL, hand
|
|
108
118
|
// control back so a person can break a non-converging loop.
|
|
109
|
-
const autoFixNow =
|
|
119
|
+
const autoFixNow = !isUnobserved
|
|
120
|
+
&& recOutcome.recommend === 'autofix'
|
|
121
|
+
&& autoFixCount < MAX_AUTO_AUTOFIX;
|
|
110
122
|
let choice;
|
|
111
123
|
if (autoFixNow) {
|
|
112
124
|
autoFixCount += 1;
|
|
@@ -21,6 +21,12 @@ export interface VerifyOutcome {
|
|
|
21
21
|
/** Short, human-readable reason. Always set when ok === false; on the pass
|
|
22
22
|
* path set to the no-op cause ('disabled', 'no spec to verify'). */
|
|
23
23
|
reason?: string;
|
|
24
|
+
/** True when the FAIL is specifically an UNOBSERVED outcome (rule 5c): a
|
|
25
|
+
* spec-required behavioral check could not run because its tooling is absent.
|
|
26
|
+
* The gate routes this straight to the human picker instead of an unattended
|
|
27
|
+
* AUTOFIX re-run, which cannot provision a missing tool. Only meaningful when
|
|
28
|
+
* ok === false. */
|
|
29
|
+
unobserved?: boolean;
|
|
24
30
|
}
|
|
25
31
|
/**
|
|
26
32
|
* Slice the delivered spec (GOAL / CONSTRAINTS / ACCEPTANCE / VERIFY) out of a
|
|
@@ -55,17 +61,24 @@ export declare function extractSpecForVerification(taskBody: string): string | n
|
|
|
55
61
|
* Guard: honest-clean fixture (prohibition in spec, probe silent) 5/5 PASS — no
|
|
56
62
|
* paranoia. Reverted-violation ≡ clean at the diff level (no entry → no finding).
|
|
57
63
|
*/
|
|
58
|
-
export declare function buildVerifyPrompt(spec: string, probeFindings?: string[], envNotes?: string, prohibitionFindings?: string[]): string;
|
|
64
|
+
export declare function buildVerifyPrompt(spec: string, probeFindings?: string[], envNotes?: string, prohibitionFindings?: string[], skipEscapeFindings?: string[], contracts?: string): string;
|
|
59
65
|
/**
|
|
60
|
-
* Parse the child's verdict. Scans for the LAST `WORK-VERIFIED: PASS|FAIL`
|
|
61
|
-
* (the model discusses before concluding, and bash output may echo the word
|
|
66
|
+
* Parse the child's verdict. Scans for the LAST `WORK-VERIFIED: PASS|FAIL|UNOBSERVED`
|
|
67
|
+
* marker (the model discusses before concluding, and bash output may echo the word
|
|
62
68
|
* "VERIFY", so a distinct token and last-match win matter).
|
|
63
69
|
*
|
|
70
|
+
* UNOBSERVED (rule 5c) is a distinct third outcome: a spec-required behavioral check
|
|
71
|
+
* could not run because its observation tooling is absent, so the behavior is neither
|
|
72
|
+
* proven nor shown broken. It is NOT a pass (`pass: false`) but carries `unobserved`
|
|
73
|
+
* so the gate can route it straight to the human — an unattended AUTOFIX re-run cannot
|
|
74
|
+
* provision a missing tool, so it must never auto-loop on this.
|
|
75
|
+
*
|
|
64
76
|
* No marker at all is NOT a pass: a verification that cannot state a verdict is a
|
|
65
77
|
* gray area, and the contract is that unverified work is reported as such.
|
|
66
78
|
*/
|
|
67
79
|
export declare function parseVerifyVerdict(text: string): {
|
|
68
80
|
pass: boolean;
|
|
81
|
+
unobserved?: boolean;
|
|
69
82
|
detail: string;
|
|
70
83
|
};
|
|
71
84
|
export interface VerificationDeps {
|
|
@@ -124,6 +137,14 @@ export interface VerificationDeps {
|
|
|
124
137
|
read: () => Promise<string>;
|
|
125
138
|
append: (notes: string[]) => Promise<void>;
|
|
126
139
|
};
|
|
140
|
+
/**
|
|
141
|
+
* Per-run cross-slice contract registry (see contracts.ts): `read` supplies the
|
|
142
|
+
* verbatim interface facts the SOURCE design pins that more than one slice
|
|
143
|
+
* touches, injected so the verify child checks THIS slice's boundary against
|
|
144
|
+
* them (F3 seam bugs are locally right but globally wrong). ABSENT/empty → no
|
|
145
|
+
* block (single `/task` runs, or a design pinning no shared boundary), unchanged.
|
|
146
|
+
*/
|
|
147
|
+
contracts?: () => Promise<string>;
|
|
127
148
|
}
|
|
128
149
|
/**
|
|
129
150
|
* Run the verification pass for one task. A missing spec is a pass. Otherwise run
|
package/dist/task/verify-work.js
CHANGED
|
@@ -69,6 +69,8 @@
|
|
|
69
69
|
*/
|
|
70
70
|
import { USER_CANCELLED } from './child-runner.js';
|
|
71
71
|
import { buildEnvNotesBlock, ENV_NOTE_EMIT_INSTRUCTION, extractEnvNotes } from './env-notes.js';
|
|
72
|
+
import { buildContractsVerifyBlock } from './contracts.js';
|
|
73
|
+
import { findSkipEscapes, skipEscapeVerifyFindings } from './skip-escape.js';
|
|
72
74
|
/**
|
|
73
75
|
* The verification child gets exactly two tools: `read` and `bash`.
|
|
74
76
|
*
|
|
@@ -137,7 +139,7 @@ export function extractSpecForVerification(taskBody) {
|
|
|
137
139
|
* Guard: honest-clean fixture (prohibition in spec, probe silent) 5/5 PASS — no
|
|
138
140
|
* paranoia. Reverted-violation ≡ clean at the diff level (no entry → no finding).
|
|
139
141
|
*/
|
|
140
|
-
export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFindings) {
|
|
142
|
+
export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFindings, skipEscapeFindings, contracts) {
|
|
141
143
|
const probeBlock = probeFindings && probeFindings.length > 0 ?
|
|
142
144
|
[
|
|
143
145
|
'SELF-VERIFICATION NOTICE (deterministic, computed by the orchestrator from the diff):',
|
|
@@ -164,7 +166,23 @@ export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFind
|
|
|
164
166
|
''
|
|
165
167
|
]
|
|
166
168
|
: [];
|
|
169
|
+
const skipEscapeBlock = skipEscapeFindings && skipEscapeFindings.length > 0 ?
|
|
170
|
+
[
|
|
171
|
+
"SKIP-ESCAPE NOTICE (deterministic, computed by the orchestrator from the spec's",
|
|
172
|
+
'OWN VERIFY block): these VERIFY commands wrap a required check in a fallback that',
|
|
173
|
+
'ANNOUNCES skipping it when a tool is absent — so the check can "pass" while never',
|
|
174
|
+
'actually running:',
|
|
175
|
+
...skipEscapeFindings.map(f => `- ${f}`),
|
|
176
|
+
'Do NOT accept a skipped check as a passed check. For each, determine whether it',
|
|
177
|
+
'ACTUALLY ran and observed the real behavior. If its tool is absent so the required',
|
|
178
|
+
'behavior was never observed, that area is UNOBSERVED (rule 5c) — verdict UNOBSERVED,',
|
|
179
|
+
'not PASS. Only if you observe the required behavior another way (running the real',
|
|
180
|
+
'artifact directly) may it count as verified.',
|
|
181
|
+
''
|
|
182
|
+
]
|
|
183
|
+
: [];
|
|
167
184
|
const envBlock = envNotes && envNotes.trim().length > 0 ? [buildEnvNotesBlock(envNotes)] : [];
|
|
185
|
+
const contractsBlock = contracts && contracts.trim().length > 0 ? [buildContractsVerifyBlock(contracts)] : [];
|
|
168
186
|
return [
|
|
169
187
|
'You are a strict verification pass running right after an AI coding agent',
|
|
170
188
|
'finished a task and committed it. The agent is known to mark work "done"',
|
|
@@ -179,8 +197,10 @@ export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFind
|
|
|
179
197
|
spec.trim(),
|
|
180
198
|
'',
|
|
181
199
|
...envBlock,
|
|
200
|
+
...contractsBlock,
|
|
182
201
|
...probeBlock,
|
|
183
202
|
...prohibitionBlock,
|
|
203
|
+
...skipEscapeBlock,
|
|
184
204
|
'How to verify — verify the REAL, shipped deliverable exactly as an unaided fresh',
|
|
185
205
|
'checkout (or CI run) would experience it:',
|
|
186
206
|
'',
|
|
@@ -242,6 +262,24 @@ export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFind
|
|
|
242
262
|
' server never starts, no entrypoint exists — that inability IS the defect:',
|
|
243
263
|
' report FAIL naming it. Do not stand up a substitute to get to green.',
|
|
244
264
|
'',
|
|
265
|
+
'3e. NEGATIVE CONTROL IS MANDATORY — a success you cannot make fail is not evidence,',
|
|
266
|
+
' and a check that only ever ran on the RIGHT input has not been shown to',
|
|
267
|
+
' discriminate working from broken. For EACH behavioral check whose passing outcome',
|
|
268
|
+
' you rely on (an endpoint returned success, a command exited 0, a call returned the',
|
|
269
|
+
' expected value) you MUST also run that SAME check with one input deliberately',
|
|
270
|
+
' wrong — a nonsense path, a bogus flag or subcommand, a malformed or absent',
|
|
271
|
+
' argument, a target that does not exist — recreating whatever setup the check needs',
|
|
272
|
+
' to do so (restart the server if a prior step killed it, re-invoke the command).',
|
|
273
|
+
' Require a DIFFERENT, failing outcome from the wrong input. If the wrong input',
|
|
274
|
+
' yields the SAME success (same status code, same body, same exit 0), the check',
|
|
275
|
+
' cannot tell working from broken: it is VOID and the behavior it was meant to prove',
|
|
276
|
+
' is UNVERIFIED — a FAIL naming the indiscriminate check (e.g. "a POST to a nonsense',
|
|
277
|
+
' path returns the same 200 and body as the real endpoint, so the route is unproven',
|
|
278
|
+
' — a catch-all fallback is masking it"). Skipping the control is not neutral: a',
|
|
279
|
+
' required behavior left without a discriminating check is UNVERIFIED, and',
|
|
280
|
+
' unverified is a FAIL, never a PASS. A wrong-input control exists for every',
|
|
281
|
+
' artifact — HTTP request, CLI invocation, library call, schema load, config parse.',
|
|
282
|
+
'',
|
|
245
283
|
'4. Treat the ACCEPTANCE criteria as the bar. If a command fails, or its real output',
|
|
246
284
|
' contradicts an ACCEPTANCE criterion, the work has NOT verified.',
|
|
247
285
|
'',
|
|
@@ -282,8 +320,36 @@ export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFind
|
|
|
282
320
|
' that is a FAIL naming the gap. Any schema surgery you performed to reach green IS the',
|
|
283
321
|
' defect.',
|
|
284
322
|
'',
|
|
285
|
-
'
|
|
286
|
-
'
|
|
323
|
+
'5c. A SPEC-REQUIRED CHECK THAT DID NOT ACTUALLY RUN IS NOT VERIFIED. The env-gap',
|
|
324
|
+
' exception (rule 5) covers ONLY a genuinely EXTERNAL service the finished product',
|
|
325
|
+
" connects to (a database, an API host). It does NOT cover a check the SPEC's own",
|
|
326
|
+
" VERIFY block authored to OBSERVE the deliverable's required behavior. If such a",
|
|
327
|
+
' check did not actually execute — the tool or runner it needs is not installed, or',
|
|
328
|
+
' the VERIFY line short-circuits ITSELF with a skip-escape (`|| true`, `|| echo',
|
|
329
|
+
' skipping`, `2>/dev/null || exit 0`, `command -v X ||` …) so a missing tool passes',
|
|
330
|
+
' silently — then that behavior was NEVER OBSERVED. A skipped check is not a passed',
|
|
331
|
+
' check: you may not count the area as verified, and "correctly skipped" is NOT a',
|
|
332
|
+
" PASS. Do not let the work's own skip-escape waive the work's own gate. When a",
|
|
333
|
+
' REQUIRED behavior could not be observed because its observation tooling is absent,',
|
|
334
|
+
' the verdict is UNOBSERVED — report exactly what could not be observed and which',
|
|
335
|
+
' tool was missing, so a human decides whether to provision the tool or accept the',
|
|
336
|
+
' unproven behavior.',
|
|
337
|
+
' DISCRIMINATOR (rule 5 vs rule 5c) — when something needed is absent, ask which it',
|
|
338
|
+
' is: (rule 5, env-gap, do NOT fail the code) a SERVICE the FINISHED PRODUCT itself',
|
|
339
|
+
' connects to at runtime to FUNCTION — a database, an API host, a message broker —',
|
|
340
|
+
' whose absence is an environment gap; versus (rule 5c, UNOBSERVED) a HARNESS that',
|
|
341
|
+
' exists only to OBSERVE or DRIVE the product during a check — a browser driver, a',
|
|
342
|
+
" UI/terminal smoke, a snapshot or fuzz tool — whose absence leaves the product's own",
|
|
343
|
+
' behavior unproven. The product NEEDS the former to work at all; it needs the latter',
|
|
344
|
+
' only to be CHECKED. A command that fails because such a runtime SERVICE is absent',
|
|
345
|
+
' stays a rule-5 env-gap and is NOT UNOBSERVED; a required behavior you could not',
|
|
346
|
+
' OBSERVE because its test harness is absent IS UNOBSERVED.',
|
|
347
|
+
'',
|
|
348
|
+
'6. If the spec legitimately has no runnable verification at all (a pure docs / config',
|
|
349
|
+
' change with nothing to build or run), validating it cleanly is a PASS. This is NOT',
|
|
350
|
+
' the same as a required behavioral check that self-skipped or whose tool is absent',
|
|
351
|
+
' (that is UNOBSERVED, rule 5c) — "nothing to verify" means the spec never demanded',
|
|
352
|
+
' an observation, not that an observation was demanded and then dodged.',
|
|
287
353
|
'',
|
|
288
354
|
'7. Do NOT edit anything to make a check pass. Report what you actually saw.',
|
|
289
355
|
'',
|
|
@@ -291,32 +357,53 @@ export function buildVerifyPrompt(spec, probeFindings, envNotes, prohibitionFind
|
|
|
291
357
|
'acceptance criterion is unmet — a required function absent, required data not persisted,',
|
|
292
358
|
'a required behavior missing — the verdict is FAIL, even if typecheck and lint are green',
|
|
293
359
|
'and even if the gap seems minor. Never downgrade an unmet criterion to a warning note.',
|
|
360
|
+
'Before concluding PASS, confirm every behavioral check you relied on has its paired',
|
|
361
|
+
'negative control (rule 3e) showing it CAN fail; a required behavior with no',
|
|
362
|
+
'discriminating check is UNVERIFIED, and the verdict is FAIL.',
|
|
294
363
|
'',
|
|
295
364
|
ENV_NOTE_EMIT_INSTRUCTION,
|
|
296
365
|
'',
|
|
297
366
|
'When you are done, output EXACTLY ONE of these as the final line:',
|
|
298
367
|
" WORK-VERIFIED: PASS (the project's own command, run unaided, met the spec)",
|
|
299
368
|
' WORK-VERIFIED: FAIL <text> (the shipped command failed or did not meet the spec; say what failed)',
|
|
369
|
+
' WORK-VERIFIED: UNOBSERVED <text> (a spec-required behavioral check could not run because',
|
|
370
|
+
' its observation tooling is absent — the behavior is',
|
|
371
|
+
' unproven, not passed and not a code failure; rule 5c)',
|
|
300
372
|
'Output the verdict line verbatim — it is parsed mechanically.'
|
|
301
373
|
].join('\n');
|
|
302
374
|
}
|
|
303
375
|
/**
|
|
304
|
-
* Parse the child's verdict. Scans for the LAST `WORK-VERIFIED: PASS|FAIL`
|
|
305
|
-
* (the model discusses before concluding, and bash output may echo the word
|
|
376
|
+
* Parse the child's verdict. Scans for the LAST `WORK-VERIFIED: PASS|FAIL|UNOBSERVED`
|
|
377
|
+
* marker (the model discusses before concluding, and bash output may echo the word
|
|
306
378
|
* "VERIFY", so a distinct token and last-match win matter).
|
|
307
379
|
*
|
|
380
|
+
* UNOBSERVED (rule 5c) is a distinct third outcome: a spec-required behavioral check
|
|
381
|
+
* could not run because its observation tooling is absent, so the behavior is neither
|
|
382
|
+
* proven nor shown broken. It is NOT a pass (`pass: false`) but carries `unobserved`
|
|
383
|
+
* so the gate can route it straight to the human — an unattended AUTOFIX re-run cannot
|
|
384
|
+
* provision a missing tool, so it must never auto-loop on this.
|
|
385
|
+
*
|
|
308
386
|
* No marker at all is NOT a pass: a verification that cannot state a verdict is a
|
|
309
387
|
* gray area, and the contract is that unverified work is reported as such.
|
|
310
388
|
*/
|
|
311
389
|
export function parseVerifyVerdict(text) {
|
|
312
|
-
const re = /WORK-VERIFIED:\s*(PASS|FAIL)\b[ \t]*(.*)/gi;
|
|
390
|
+
const re = /WORK-VERIFIED:\s*(PASS|FAIL|UNOBSERVED)\b[ \t]*(.*)/gi;
|
|
313
391
|
let last = null;
|
|
314
392
|
for (let m = re.exec(text); m !== null; m = re.exec(text))
|
|
315
393
|
last = m;
|
|
316
394
|
if (!last)
|
|
317
395
|
return { pass: false, detail: 'no verdict emitted' };
|
|
318
|
-
const
|
|
319
|
-
|
|
396
|
+
const kind = last[1].toUpperCase();
|
|
397
|
+
if (kind === 'PASS')
|
|
398
|
+
return { pass: true, detail: '' };
|
|
399
|
+
if (kind === 'UNOBSERVED') {
|
|
400
|
+
return {
|
|
401
|
+
pass: false,
|
|
402
|
+
unobserved: true,
|
|
403
|
+
detail: last[2].trim() || 'a required behavior could not be observed'
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
return { pass: false, detail: last[2].trim() || 'unspecified failure' };
|
|
320
407
|
}
|
|
321
408
|
/**
|
|
322
409
|
* Run the verification pass for one task. A missing spec is a pass. Otherwise run
|
|
@@ -370,6 +457,23 @@ export async function runWorkVerification(deps) {
|
|
|
370
457
|
envNotes = '';
|
|
371
458
|
}
|
|
372
459
|
}
|
|
460
|
+
// Cross-slice contracts from decompose-time extraction (best-effort; a read
|
|
461
|
+
// fault must never block verification).
|
|
462
|
+
let contracts = '';
|
|
463
|
+
if (deps.contracts) {
|
|
464
|
+
try {
|
|
465
|
+
contracts = await deps.contracts();
|
|
466
|
+
}
|
|
467
|
+
catch {
|
|
468
|
+
contracts = '';
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
// DETERMINISTIC skip-escape finding, computed purely from the spec's own VERIFY
|
|
472
|
+
// block (see skip-escape.ts): a required check wrapped in a skip-announcing `||`
|
|
473
|
+
// fallback. Injected so rule 5c fires reliably — the model does not self-discover a
|
|
474
|
+
// graceful skip-escape (A/B: rule alone ~1-3/5), but acts on a finding naming the
|
|
475
|
+
// exact line, per the proven probe+rule pattern. Pure text analysis, no dep needed.
|
|
476
|
+
const skipEscapes = skipEscapeVerifyFindings(findSkipEscapes(deps.spec));
|
|
373
477
|
// A child that emits NO verdict never judged the work (budget/context death mid-
|
|
374
478
|
// investigation — seen live: an 11-minute verify wandered, died verdict-less, and
|
|
375
479
|
// the resulting FAIL burned a full implementation re-run on an unjudged artifact).
|
|
@@ -377,7 +481,7 @@ export async function runWorkVerification(deps) {
|
|
|
377
481
|
for (let attempt = 1;; attempt++) {
|
|
378
482
|
let text;
|
|
379
483
|
try {
|
|
380
|
-
text = await deps.runChild(VERIFY_TOOLS, buildVerifyPrompt(deps.spec, findings, envNotes, prohibitions), deps.signal);
|
|
484
|
+
text = await deps.runChild(VERIFY_TOOLS, buildVerifyPrompt(deps.spec, findings, envNotes, prohibitions, skipEscapes, contracts), deps.signal);
|
|
381
485
|
}
|
|
382
486
|
catch (err) {
|
|
383
487
|
if (err instanceof Error && err.message === USER_CANCELLED)
|
|
@@ -414,6 +518,12 @@ export async function runWorkVerification(deps) {
|
|
|
414
518
|
return { ok: true };
|
|
415
519
|
if (verdict.detail === 'no verdict emitted' && attempt === 1)
|
|
416
520
|
continue;
|
|
521
|
+
// UNOBSERVED (rule 5c): a spec-required behavioral check could not run because
|
|
522
|
+
// its tooling is absent. Block like any FAIL, but flag it so the gate hands it
|
|
523
|
+
// straight to the human — re-running the impl turn cannot install a missing tool.
|
|
524
|
+
if (verdict.unobserved) {
|
|
525
|
+
return { ok: false, unobserved: true, reason: `work unobserved: ${verdict.detail}` };
|
|
526
|
+
}
|
|
417
527
|
return {
|
|
418
528
|
ok: false,
|
|
419
529
|
reason: `work did not verify: ${verdict.detail}${verdict.detail === 'no verdict emitted' ? ' (after verify retry)' : ''}`
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic synthesized-wiring scanner for a composed spec (run-8 F3, gen side).
|
|
3
|
+
*
|
|
4
|
+
* F3 (the dominant run-8 shipped defect): refine/compose invent a "uniform" wiring
|
|
5
|
+
* table — one module → one mount prefix, `/api/<x>` → `<x>Routes` for every module —
|
|
6
|
+
* though the design pins ENDPOINTS, not mounts, and one module's pinned endpoints do
|
|
7
|
+
* NOT all sit under a single prefix (photos: `POST /api/listings/:id/photos` AND
|
|
8
|
+
* `GET/DELETE /api/photos/:id`). Mounting that module at `/api/photos` double-prefixes
|
|
9
|
+
* the upload; consumers follow the pinned paths, assembly follows the invented table,
|
|
10
|
+
* the seam ships broken. See [[contract-registry-f3]] (#4, the verify-side + registry
|
|
11
|
+
* lever) — this is its GENERATION-side complement.
|
|
12
|
+
*
|
|
13
|
+
* A/B-measured on the live 27B (F3 critique trap): the CROSS-SLICE CONTRACTS registry
|
|
14
|
+
* is NECESSARY but the prompt+registry alone is a WEAK catcher (A/B arms 0/8, +registry
|
|
15
|
+
* only 1/8) — the model's attention goes to the obvious VERIFY weakness and it rarely
|
|
16
|
+
* does the path-composition reasoning even with the facts in front of it. The reliable
|
|
17
|
+
* lever is the SAME probe+rule pattern as [[skip-escape-scanner-f2]] / [[verify-
|
|
18
|
+
* substitution-ab]]: a deterministic finding that NAMES the exact synthesized mappings
|
|
19
|
+
* and juxtaposes the verbatim pinned facts, forcing focused reconciliation.
|
|
20
|
+
*
|
|
21
|
+
* This scanner does NOT decide which mapping is wrong — that needs routing-composition
|
|
22
|
+
* knowledge (a forbidden stack assumption). It surfaces every mapping that (a) is not a
|
|
23
|
+
* verbatim substring of the design/registry (so it is INFERRED, not cited) AND (b) touches
|
|
24
|
+
* a pinned cross-slice boundary (an operand appears in the registry) — i.e. it reshapes a
|
|
25
|
+
* shared contract. The 4 coincidentally-correct mappings are surfaced too, but framed as
|
|
26
|
+
* "reconcile each; keep the conforming ones" — the LLM decides, informed. Pure text/
|
|
27
|
+
* substring analysis; no stack, framework, or routing assumptions. Empty registry (single
|
|
28
|
+
* `/task`, or no shared boundary) ⇒ no-op.
|
|
29
|
+
*/
|
|
30
|
+
export interface WiringClaim {
|
|
31
|
+
/** The offending mapping line, verbatim (trimmed). */
|
|
32
|
+
line: string;
|
|
33
|
+
/** Left operand (the mapped-from token, e.g. a mount prefix). */
|
|
34
|
+
from: string;
|
|
35
|
+
/** Right operand (the mapped-to token, e.g. a module name). */
|
|
36
|
+
to: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Find synthesized wiring mappings in `spec`: `A <arrow> B` lines whose whole mapping
|
|
40
|
+
* is NOT a verbatim substring of `grounding` (design ∪ registry) yet an operand appears
|
|
41
|
+
* in `registry` (so it reshapes a pinned cross-slice boundary). Returns [] when the
|
|
42
|
+
* registry is empty (no shared contracts to reshape) — the whole check is a no-op then.
|
|
43
|
+
*/
|
|
44
|
+
export declare function findSynthesizedWiring(spec: string, grounding: string, registry: string): WiringClaim[];
|
|
45
|
+
/**
|
|
46
|
+
* Render findings as the critique probe (probe+rule pattern): NAME the inferred
|
|
47
|
+
* mappings and juxtapose the verbatim pinned facts, then instruct focused
|
|
48
|
+
* reconciliation. Deliberately does NOT accuse a specific mapping — the model decides
|
|
49
|
+
* which (if any) fails to reproduce a pinned fact. Empty findings ⇒ '' (no block).
|
|
50
|
+
*/
|
|
51
|
+
export declare function wiringProbeText(findings: WiringClaim[], registry: string): string;
|
|
52
|
+
/**
|
|
53
|
+
* Render findings as a critique-rewrite defect block (fed into the FOCUS list): the
|
|
54
|
+
* rewrite must reconcile each mapping against the pinned facts and correct only the
|
|
55
|
+
* one(s) that break. Mirrors skipEscapeDefectText's shape.
|
|
56
|
+
*/
|
|
57
|
+
export declare function wiringDefectText(findings: WiringClaim[], registry: string): string;
|
|
58
|
+
/**
|
|
59
|
+
* Concatenate the design/spec docs the given texts @-reference (best-effort, readable
|
|
60
|
+
* files only) as extra grounding for findSynthesizedWiring — so a mapping the design
|
|
61
|
+
* states verbatim is treated as CITED, not synthesized. Unreadable/absent mentions are
|
|
62
|
+
* skipped; returns '' when nothing resolves.
|
|
63
|
+
*/
|
|
64
|
+
export declare function readReferencedDocs(cwd: string, ...texts: string[]): string;
|