@awak-app/simy-cli 0.5.10 → 0.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/package.json +2 -2
- package/src/cli-contract.js +1 -1
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/SKILL.md +90 -10
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/references/acceptance-evidence.md +28 -3
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/references/delivery-levels.md +5 -1
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/references/minimum-completion.md +146 -0
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/references/shared-system-specifications.md +151 -0
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/references/state-and-receipts.md +8 -0
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/scripts/check-acceptance.mjs +28 -8
- package/src/codex-plugins/simy/skills/agentic-delivery-loop/scripts/check-minimum-completion.mjs +273 -0
- package/src/orchestrator/audit.js +17 -2
- package/src/orchestrator/completion-contract.js +51 -5
- package/src/orchestrator/contract.js +12 -1
- package/src/orchestrator/evidence.js +9 -1
- package/src/orchestrator/independent-audit.js +17 -4
- package/src/orchestrator/instruction.js +20 -2
- package/src/orchestrator/loop.js +34 -4
- package/src/orchestrator/minimum-completion.js +108 -0
- package/src/orchestrator/result.js +7 -0
package/src/codex-plugins/simy/skills/agentic-delivery-loop/references/state-and-receipts.md
CHANGED
|
@@ -9,6 +9,8 @@ Persist one controller-owned snapshot with at least:
|
|
|
9
9
|
|
|
10
10
|
- `run_id`, `state`, monotonic `revision`, and terminal disposition;
|
|
11
11
|
- immutable explicit conditions and their stable digest;
|
|
12
|
+
- frozen experience-contract identities, their mapping to acceptance rows, and
|
|
13
|
+
the current readable local-experience table path and digest;
|
|
12
14
|
- acceptance-source identities and digests, architecture boundaries, applicable
|
|
13
15
|
rule decisions, risk tier, requested/effective delivery levels, the safety
|
|
14
16
|
ceiling reason, and chat-confirmation receipt;
|
|
@@ -135,6 +137,7 @@ For the full delivery mode:
|
|
|
135
137
|
done =
|
|
136
138
|
reproduction_verified
|
|
137
139
|
&& acceptance_evidence_complete
|
|
140
|
+
&& minimum_functional_evidence_verified
|
|
138
141
|
&& locally_verified
|
|
139
142
|
&& pr_ci_green
|
|
140
143
|
&& integration_verified
|
|
@@ -155,3 +158,8 @@ with current context and the required user/system layers. After each evaluated
|
|
|
155
158
|
action, refresh the JSON matrix and its readable table; append a receipt for the
|
|
156
159
|
update. Keep the frozen charter distinct from mutable results. A charter patch
|
|
157
160
|
supersedes the old digest only after the controller verifies its authority.
|
|
161
|
+
For `user_like_local_verified`, the current table must cover the frozen experience
|
|
162
|
+
contract from real trigger to usable outcome for the exact local candidate. A
|
|
163
|
+
collection of narrower passing rows does not waive a missing end-to-end row.
|
|
164
|
+
|
|
165
|
+
The CLI records minimum evidence in observed_evidence.minimum_completion, independently of executor status. The independent audit is bound to its matrix digest. Legacy v2 histories remain readable, but missing minimum evidence cannot be reused as passing evidence on resume/recheck. Missing or unsupported charters require a refresh; they do not disable verification.
|
|
@@ -25,7 +25,15 @@ function requireValue(condition, message) {
|
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
function validateCharter(charter) {
|
|
28
|
-
requireValue(object(charter) && charter.schema_version
|
|
28
|
+
requireValue(object(charter) && [1, 2].includes(charter.schema_version) && text(charter.run_id), "Invalid charter header");
|
|
29
|
+
if (charter.schema_version === 2) {
|
|
30
|
+
requireValue(object(charter.experience), "Experience contract required");
|
|
31
|
+
for (const field of ["beneficiary", "situation", "goal", "usable_outcome", "recovery"])
|
|
32
|
+
requireValue(text(charter.experience[field]), `Experience contract needs ${field}`);
|
|
33
|
+
requireValue(Array.isArray(charter.experience.forbidden_experiences) &&
|
|
34
|
+
charter.experience.forbidden_experiences.length > 0 &&
|
|
35
|
+
charter.experience.forbidden_experiences.every(text), "Experience contract needs forbidden experiences");
|
|
36
|
+
}
|
|
29
37
|
requireValue(Array.isArray(charter.criteria) && charter.criteria.length > 0, "Charter requires criteria");
|
|
30
38
|
const ids = new Set();
|
|
31
39
|
for (const criterion of charter.criteria) {
|
|
@@ -60,7 +68,7 @@ function validateContext(context, runId) {
|
|
|
60
68
|
}
|
|
61
69
|
}
|
|
62
70
|
|
|
63
|
-
async function
|
|
71
|
+
export async function readEvidenceArtifact(root, artifact) {
|
|
64
72
|
requireValue(object(artifact) && text(artifact.path) && hashPattern.test(artifact.sha256), "Invalid artifact reference");
|
|
65
73
|
// Explicit bundle-relative paths only; never fetch URLs or execute evidence.
|
|
66
74
|
const segments = artifact.path.split("/");
|
|
@@ -77,7 +85,9 @@ async function checkArtifact(root, artifact) {
|
|
|
77
85
|
try {
|
|
78
86
|
const stat = await handle.stat();
|
|
79
87
|
requireValue(stat.isFile() && stat.size > 0 && stat.size <= 16 * 1024 * 1024, "Artifact must be a nonempty regular file of at most 16 MiB");
|
|
80
|
-
|
|
88
|
+
const bytes = await handle.readFile();
|
|
89
|
+
requireValue(sha256(bytes) === artifact.sha256, "Artifact digest mismatch");
|
|
90
|
+
return bytes;
|
|
81
91
|
} finally {
|
|
82
92
|
await handle.close();
|
|
83
93
|
}
|
|
@@ -86,15 +96,24 @@ async function checkArtifact(root, artifact) {
|
|
|
86
96
|
const tableText = (value) => String(value ?? "").replaceAll("|", "\\|").replace(/[\r\n]+/g, " ");
|
|
87
97
|
|
|
88
98
|
export function renderChecklist(report) {
|
|
99
|
+
const experienceContext = report.experience
|
|
100
|
+
? `${report.experience.beneficiary} / ${report.experience.situation}`
|
|
101
|
+
: "Legacy charter: persona and situation not recorded";
|
|
89
102
|
const header = [
|
|
90
|
-
"|
|
|
91
|
-
"| --- | --- | --- | --- | --- | --- |",
|
|
103
|
+
"| Experience criterion / expected outcome | Persona / situation | Layer / method | Status | Actual observation | Evidence | Unmet reason / next repair |",
|
|
104
|
+
"| --- | --- | --- | --- | --- | --- | --- |",
|
|
92
105
|
];
|
|
93
106
|
const rows = report.checks.map((row) => "| " + [
|
|
94
|
-
`${row.id}: ${row.outcome}`,
|
|
107
|
+
`${row.id}: ${row.outcome}`, experienceContext,
|
|
108
|
+
`${row.layer}: ${row.method}`, row.status,
|
|
95
109
|
row.observation, row.evidence.join(", "), [row.reason, row.next_action].filter(Boolean).join(" / "),
|
|
96
110
|
].map(tableText).join(" | ") + " |");
|
|
111
|
+
const context = report.context;
|
|
112
|
+
const candidate = context
|
|
113
|
+
? `Candidate context: stage=${tableText(context.stage)}; environment=${tableText(context.environment)}; fixture=${tableText(context.fixture_id)}; input_sha256=${tableText(context.input_sha256)}; source_revisions=${tableText(JSON.stringify(context.source_revisions))}`
|
|
114
|
+
: "Candidate context: unavailable";
|
|
97
115
|
return [...header, ...rows, "", `Validator: ${report.status}`,
|
|
116
|
+
candidate,
|
|
98
117
|
...report.errors.map((error) => `- ${tableText(error)}`)].join("\n") + "\n";
|
|
99
118
|
}
|
|
100
119
|
|
|
@@ -109,7 +128,8 @@ export async function checkAcceptance({ charterBytes, matrix, expectedCharterSha
|
|
|
109
128
|
requireValue(contextDigest(matrix.context) === expectedContextSha256, "Current context digest mismatch");
|
|
110
129
|
requireValue(Array.isArray(matrix.results), "Matrix requires results");
|
|
111
130
|
const root = await realpath(evidenceRoot);
|
|
112
|
-
const report = { status: "evidence_incomplete",
|
|
131
|
+
const report = { status: "evidence_incomplete", experience: charter.experience ?? null,
|
|
132
|
+
context: matrix.context, checks: [], errors: [] };
|
|
113
133
|
const results = new Map();
|
|
114
134
|
const expectedIds = new Set(charter.criteria.map((criterion) => criterion.id));
|
|
115
135
|
for (const row of matrix.results) {
|
|
@@ -150,7 +170,7 @@ export async function checkAcceptance({ charterBytes, matrix, expectedCharterSha
|
|
|
150
170
|
requireValue(text(cell.observed_at) && /^\d{4}-\d{2}-\d{2}T.*(?:Z|[+-]\d{2}:\d{2})$/.test(cell.observed_at) &&
|
|
151
171
|
Number.isFinite(Date.parse(cell.observed_at)) && Date.parse(cell.observed_at) <= Date.now(), "Invalid or future observation timestamp");
|
|
152
172
|
requireValue(Array.isArray(cell.evidence) && cell.evidence.length > 0, "Passed result needs evidence");
|
|
153
|
-
for (const artifact of cell.evidence) await
|
|
173
|
+
for (const artifact of cell.evidence) await readEvidenceArtifact(root, artifact);
|
|
154
174
|
row.status = "passed";
|
|
155
175
|
} catch {
|
|
156
176
|
row.reason = "Invalid, missing, empty, unsafe, or changed evidence; check action, observation, timestamp and artifact hashes";
|
package/src/codex-plugins/simy/skills/agentic-delivery-loop/scripts/check-minimum-completion.mjs
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Record/observation checks, not authentication of the collector or host completion.
|
|
3
|
+
import { lstat, readFile, realpath } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { isDeepStrictEqual } from "node:util";
|
|
7
|
+
import {
|
|
8
|
+
checkAcceptance,
|
|
9
|
+
contextDigest,
|
|
10
|
+
readEvidenceArtifact,
|
|
11
|
+
sha256,
|
|
12
|
+
} from "./check-acceptance.mjs";
|
|
13
|
+
|
|
14
|
+
const text = (value) => typeof value === "string" && value.trim().length > 0;
|
|
15
|
+
const object = (value) =>
|
|
16
|
+
value !== null && typeof value === "object" && !Array.isArray(value);
|
|
17
|
+
export const MINIMUM_COMPLETION_VERSION = 1;
|
|
18
|
+
|
|
19
|
+
export function minimumCharter(charter) {
|
|
20
|
+
if (
|
|
21
|
+
charter?.completion_contract?.version !== 2 ||
|
|
22
|
+
!text(charter.id) ||
|
|
23
|
+
!text(charter.requirement) ||
|
|
24
|
+
!Array.isArray(charter.acceptance_criteria) ||
|
|
25
|
+
!charter.acceptance_criteria.length ||
|
|
26
|
+
!charter.acceptance_criteria.every(text)
|
|
27
|
+
) {
|
|
28
|
+
throw new Error("Current completion charter required");
|
|
29
|
+
}
|
|
30
|
+
const source = {
|
|
31
|
+
original_request:
|
|
32
|
+
charter.thread_state?.original_request || charter.requirement,
|
|
33
|
+
requirement: charter.requirement,
|
|
34
|
+
minimum_scope:
|
|
35
|
+
charter.completion_contract.minimum_scope || charter.acceptance_criteria,
|
|
36
|
+
non_goals: charter.completion_contract.non_goals || [],
|
|
37
|
+
};
|
|
38
|
+
const ids = criterionIds(
|
|
39
|
+
charter.completion_contract.criterion_ids,
|
|
40
|
+
charter.acceptance_criteria.length,
|
|
41
|
+
);
|
|
42
|
+
const experience = minimumExperience(
|
|
43
|
+
charter.completion_contract.experience,
|
|
44
|
+
charter.completion_contract.user_outcome || charter.requirement,
|
|
45
|
+
);
|
|
46
|
+
return {
|
|
47
|
+
schema_version: 2,
|
|
48
|
+
minimum_completion_version: MINIMUM_COMPLETION_VERSION,
|
|
49
|
+
run_id: charter.id,
|
|
50
|
+
source,
|
|
51
|
+
experience,
|
|
52
|
+
criteria: charter.acceptance_criteria.map((outcome, index) => ({
|
|
53
|
+
id: ids[index],
|
|
54
|
+
outcome,
|
|
55
|
+
acceptance_source: `request:${contextDigest(source)}`,
|
|
56
|
+
layers: {
|
|
57
|
+
user: {
|
|
58
|
+
applicability: "required",
|
|
59
|
+
method:
|
|
60
|
+
"Exercise the actual operation and observe its result; reopen/reload when persistence is required.",
|
|
61
|
+
},
|
|
62
|
+
system: {
|
|
63
|
+
applicability: "required",
|
|
64
|
+
method:
|
|
65
|
+
"Independently inspect the final postcondition and necessary integration; persisted values and ownership when saving.",
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
})),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function criterionIds(value, count) {
|
|
73
|
+
if (value === undefined || value === null)
|
|
74
|
+
return Array.from({ length: count }, (_, index) => `AC-${index + 1}`);
|
|
75
|
+
if (!Array.isArray(value)) throw new Error("Invalid acceptance criterion IDs");
|
|
76
|
+
const ids = value.map((id) => (text(id) ? id.trim() : ""));
|
|
77
|
+
if (
|
|
78
|
+
ids.length !== count ||
|
|
79
|
+
new Set(ids).size !== ids.length ||
|
|
80
|
+
ids.some(
|
|
81
|
+
(id) => !/^[A-Za-z][A-Za-z0-9]*(?:[-_.][A-Za-z0-9]+)*$/.test(id),
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
throw new Error("Invalid acceptance criterion IDs");
|
|
85
|
+
return ids;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function minimumExperience(value, userOutcome) {
|
|
89
|
+
const source = object(value) ? value : {};
|
|
90
|
+
return {
|
|
91
|
+
beneficiary: text(source.beneficiary)
|
|
92
|
+
? source.beneficiary.trim()
|
|
93
|
+
: "The requester",
|
|
94
|
+
situation: text(source.situation)
|
|
95
|
+
? source.situation.trim()
|
|
96
|
+
: "Using the requested changed behavior",
|
|
97
|
+
goal: text(source.goal) ? source.goal.trim() : userOutcome,
|
|
98
|
+
usable_outcome: text(source.usable_outcome)
|
|
99
|
+
? source.usable_outcome.trim()
|
|
100
|
+
: userOutcome,
|
|
101
|
+
recovery: text(source.recovery)
|
|
102
|
+
? source.recovery.trim()
|
|
103
|
+
: "Unmet acceptance rows remain visible with the next repair.",
|
|
104
|
+
forbidden_experiences:
|
|
105
|
+
Array.isArray(source.forbidden_experiences) &&
|
|
106
|
+
source.forbidden_experiences.length > 0 &&
|
|
107
|
+
source.forbidden_experiences.every(text)
|
|
108
|
+
? source.forbidden_experiences.map((item) => item.trim())
|
|
109
|
+
: [
|
|
110
|
+
"Reporting completion from components or commands without the end-to-end outcome",
|
|
111
|
+
],
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function minimumContext(charter, headSha) {
|
|
116
|
+
const frozen = minimumCharter(charter);
|
|
117
|
+
return {
|
|
118
|
+
run_id: charter.id,
|
|
119
|
+
stage: "locally_verified",
|
|
120
|
+
environment: "local",
|
|
121
|
+
fixture_id: charter.id,
|
|
122
|
+
input_sha256: contextDigest(frozen.source),
|
|
123
|
+
source_revisions: { [charter.repository]: headSha },
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export async function loadMinimumJson(file) {
|
|
128
|
+
const stat = await lstat(file);
|
|
129
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 1024 * 1024)
|
|
130
|
+
throw new Error("Invalid minimum evidence input");
|
|
131
|
+
return JSON.parse(await readFile(file, "utf8"));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export async function checkMinimumCompletion({
|
|
135
|
+
charter,
|
|
136
|
+
matrix,
|
|
137
|
+
context,
|
|
138
|
+
evidenceRoot,
|
|
139
|
+
}) {
|
|
140
|
+
const frozen = minimumCharter(charter);
|
|
141
|
+
const charterBytes = Buffer.from(JSON.stringify(frozen));
|
|
142
|
+
const charterSha = sha256(charterBytes);
|
|
143
|
+
const contextSha = contextDigest(context);
|
|
144
|
+
const report = await checkAcceptance({
|
|
145
|
+
charterBytes,
|
|
146
|
+
matrix,
|
|
147
|
+
expectedCharterSha256: charterSha,
|
|
148
|
+
expectedContextSha256: contextSha,
|
|
149
|
+
evidenceRoot,
|
|
150
|
+
});
|
|
151
|
+
report.minimum_completion_version = MINIMUM_COMPLETION_VERSION;
|
|
152
|
+
report.charter_sha256 = charterSha;
|
|
153
|
+
report.context_sha256 = contextSha;
|
|
154
|
+
report.matrix_sha256 = contextDigest(matrix);
|
|
155
|
+
report.head_sha = context.source_revisions[charter.repository];
|
|
156
|
+
const fail = (id, reason) => report.errors.push(`${id}: ${reason}`);
|
|
157
|
+
for (const criterion of frozen.criteria) {
|
|
158
|
+
const row = matrix.results.find((result) => result.id === criterion.id);
|
|
159
|
+
if (
|
|
160
|
+
!Array.isArray(row?.implementation_refs) ||
|
|
161
|
+
!row.implementation_refs.length ||
|
|
162
|
+
!row.implementation_refs.every(
|
|
163
|
+
(ref) =>
|
|
164
|
+
text(ref) &&
|
|
165
|
+
!path.isAbsolute(ref) &&
|
|
166
|
+
!ref.includes("\\") &&
|
|
167
|
+
ref
|
|
168
|
+
.split("/")
|
|
169
|
+
.every(
|
|
170
|
+
(segment) =>
|
|
171
|
+
segment &&
|
|
172
|
+
segment !== ".." &&
|
|
173
|
+
segment !== "." &&
|
|
174
|
+
!segment.includes(":"),
|
|
175
|
+
),
|
|
176
|
+
)
|
|
177
|
+
) {
|
|
178
|
+
fail(
|
|
179
|
+
criterion.id,
|
|
180
|
+
"Missing or unsafe requirement-to-implementation mapping",
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
for (const [layer, names] of Object.entries({
|
|
184
|
+
user: ["operation", "result"],
|
|
185
|
+
system: ["postcondition", "integration"],
|
|
186
|
+
})) {
|
|
187
|
+
const cell = row?.layers?.[layer];
|
|
188
|
+
if (cell?.status !== "passed") continue;
|
|
189
|
+
try {
|
|
190
|
+
// The first artifact is a structured observation; further artifacts may be images/logs.
|
|
191
|
+
const observation = JSON.parse(
|
|
192
|
+
(
|
|
193
|
+
await readEvidenceArtifact(
|
|
194
|
+
await realpath(evidenceRoot),
|
|
195
|
+
cell.evidence[0],
|
|
196
|
+
)
|
|
197
|
+
).toString(),
|
|
198
|
+
);
|
|
199
|
+
if (
|
|
200
|
+
observation.criterion_id !== criterion.id ||
|
|
201
|
+
observation.context_sha256 !== contextSha ||
|
|
202
|
+
!object(observation.checks)
|
|
203
|
+
)
|
|
204
|
+
throw new Error("Wrong observation context");
|
|
205
|
+
for (const name of names) {
|
|
206
|
+
const check = observation.checks[name];
|
|
207
|
+
if (
|
|
208
|
+
!object(check) ||
|
|
209
|
+
!text(check.subject) ||
|
|
210
|
+
check.expected === null ||
|
|
211
|
+
check.expected === undefined ||
|
|
212
|
+
(name === "operation" && check.expected !== true) ||
|
|
213
|
+
!isDeepStrictEqual(check.expected, check.actual)
|
|
214
|
+
) {
|
|
215
|
+
fail(
|
|
216
|
+
`${criterion.id}/${layer}/${name}`,
|
|
217
|
+
"Missing observation or expected/actual mismatch",
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
} catch {
|
|
222
|
+
fail(
|
|
223
|
+
`${criterion.id}/${layer}`,
|
|
224
|
+
"Missing, altered or invalid structured observation",
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
report.status = report.errors.length
|
|
230
|
+
? "evidence_incomplete"
|
|
231
|
+
: "evidence_complete";
|
|
232
|
+
// Additional floor failures must be visible in the human-readable record too.
|
|
233
|
+
report.markdown += `\nMinimum completion: ${report.status}\n${report.errors.map((error) => `- ${error}`).join("\n")}\n`;
|
|
234
|
+
return report;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const entry = process.argv[1]
|
|
238
|
+
? await realpath(process.argv[1]).catch(() => null)
|
|
239
|
+
: null;
|
|
240
|
+
if (entry === fileURLToPath(import.meta.url)) {
|
|
241
|
+
try {
|
|
242
|
+
const [
|
|
243
|
+
charterPath,
|
|
244
|
+
matrixPath,
|
|
245
|
+
contextPath,
|
|
246
|
+
expectedCharter,
|
|
247
|
+
expectedContext,
|
|
248
|
+
...extra
|
|
249
|
+
] = process.argv.slice(2);
|
|
250
|
+
if (!expectedContext || extra.length) throw new Error("Invalid arguments");
|
|
251
|
+
const charter = await loadMinimumJson(charterPath);
|
|
252
|
+
const context = await loadMinimumJson(contextPath);
|
|
253
|
+
if (
|
|
254
|
+
sha256(Buffer.from(JSON.stringify(minimumCharter(charter)))) !==
|
|
255
|
+
expectedCharter ||
|
|
256
|
+
contextDigest(context) !== expectedContext
|
|
257
|
+
)
|
|
258
|
+
throw new Error("Trusted digest mismatch");
|
|
259
|
+
const report = await checkMinimumCompletion({
|
|
260
|
+
charter,
|
|
261
|
+
context,
|
|
262
|
+
matrix: await loadMinimumJson(matrixPath),
|
|
263
|
+
evidenceRoot: path.dirname(path.resolve(matrixPath)),
|
|
264
|
+
});
|
|
265
|
+
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
266
|
+
process.exitCode = report.status === "evidence_complete" ? 0 : 1;
|
|
267
|
+
} catch {
|
|
268
|
+
process.stderr.write(
|
|
269
|
+
"Minimum completion unavailable. Supply charter.json matrix.json context.json trusted-charter-sha256 trusted-context-sha256.\n",
|
|
270
|
+
);
|
|
271
|
+
process.exitCode = 2;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { minimumCompletionCheck } from "./minimum-completion.js";
|
|
1
2
|
import { open, readdir, stat } from "node:fs/promises";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
|
|
@@ -131,7 +132,7 @@ export async function auditAttempt(charter, attempt, { previousAttempt = null, s
|
|
|
131
132
|
}
|
|
132
133
|
attempt.retry_problem_solving = retryProblemSolving;
|
|
133
134
|
|
|
134
|
-
for (const check of completionEvidenceChecks(charter, attempt)) {
|
|
135
|
+
for (const check of [...completionEvidenceChecks(charter, attempt), minimumCompletionCheck(charter, attempt)]) {
|
|
135
136
|
addCheck(
|
|
136
137
|
check.id,
|
|
137
138
|
check.label,
|
|
@@ -143,6 +144,13 @@ export async function auditAttempt(charter, attempt, { previousAttempt = null, s
|
|
|
143
144
|
);
|
|
144
145
|
}
|
|
145
146
|
|
|
147
|
+
if (attempt.independent_audit) {
|
|
148
|
+
addCheck("minimum_review_current", "Independent review covers the current evidence",
|
|
149
|
+
Boolean(attempt.independent_audit.minimum_matrix_sha256 &&
|
|
150
|
+
attempt.independent_audit.minimum_matrix_sha256 === attempt.observed_evidence?.minimum_completion?.matrix_sha256),
|
|
151
|
+
"Changed minimum evidence requires a fresh independent review.");
|
|
152
|
+
}
|
|
153
|
+
|
|
146
154
|
const evidenceRequired = charter.expected_evidence.includes("ui_evidence_path");
|
|
147
155
|
const evidenceArtifact = await inspectEvidence(charter, attempt.ui_evidence_path);
|
|
148
156
|
if (evidenceRequired || attempt.ui_evidence_path) {
|
|
@@ -305,7 +313,12 @@ export function evaluatePrReadiness(charter, attempt) {
|
|
|
305
313
|
const github = attempt.observed_evidence?.github || {};
|
|
306
314
|
const local = attempt.observed_evidence?.local || {};
|
|
307
315
|
const independentAudit = attempt.independent_audit;
|
|
316
|
+
const minimum = minimumCompletionCheck(charter, attempt);
|
|
308
317
|
const checks = [
|
|
318
|
+
readinessCheck(minimum.id, minimum.passed, minimum.detail),
|
|
319
|
+
readinessCheck("minimum_independent_binding",
|
|
320
|
+
Boolean(independentAudit?.minimum_matrix_sha256 && independentAudit.minimum_matrix_sha256 === attempt.observed_evidence?.minimum_completion?.matrix_sha256),
|
|
321
|
+
"Independent review must cover the current minimum evidence matrix."),
|
|
309
322
|
readinessCheck("implementation_gate", attempt.audit?.passed === true, "Implementation gate must pass."),
|
|
310
323
|
readinessCheck(
|
|
311
324
|
"independent_ai_audit",
|
|
@@ -372,7 +385,9 @@ export function evaluatePrReadiness(charter, attempt) {
|
|
|
372
385
|
state: ready ? "merge_ready" : "pr_ready_for_review",
|
|
373
386
|
summary: ready
|
|
374
387
|
? "PR has independently verified evidence, passing checks, and required approval."
|
|
375
|
-
:
|
|
388
|
+
: attempt.audit?.passed && minimum.passed
|
|
389
|
+
? "Implementation is verified, but the PR is not merge-ready."
|
|
390
|
+
: "Implementation verification is incomplete; resolve the pending evidence before promotion.",
|
|
376
391
|
checks,
|
|
377
392
|
pending_reasons: checks.filter((check) => !check.passed).map((check) => check.detail),
|
|
378
393
|
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { cleanString, stringArray } from "./shared.js";
|
|
2
2
|
|
|
3
3
|
export const COMPLETION_CONTRACT_VERSION = 2;
|
|
4
|
+
const criterionIdPattern = /^[A-Za-z][A-Za-z0-9]*(?:[-_.][A-Za-z0-9]+)*$/;
|
|
4
5
|
|
|
5
6
|
export function buildCompletionContract({
|
|
6
7
|
inherited,
|
|
@@ -12,11 +13,15 @@ export function buildCompletionContract({
|
|
|
12
13
|
const source = record(inherited);
|
|
13
14
|
const minimumScope = stringArray(source.minimum_scope);
|
|
14
15
|
const proofRequired = stringArray(source.proof_required);
|
|
16
|
+
const criteria = stringArray(acceptanceCriteria);
|
|
17
|
+
const userOutcome = cleanString(source.user_outcome) || cleanString(requirement);
|
|
15
18
|
return {
|
|
16
19
|
version: COMPLETION_CONTRACT_VERSION,
|
|
17
|
-
user_outcome:
|
|
20
|
+
user_outcome: userOutcome,
|
|
18
21
|
minimum_scope:
|
|
19
|
-
minimumScope.length > 0 ? minimumScope :
|
|
22
|
+
minimumScope.length > 0 ? minimumScope : criteria,
|
|
23
|
+
criterion_ids: criterionIds(source.criterion_ids, criteria.length),
|
|
24
|
+
experience: normalizeExperience(source.experience, userOutcome),
|
|
20
25
|
non_goals: stringArray(nonGoals),
|
|
21
26
|
proof_required: uniqueStrings([
|
|
22
27
|
"criterion_evidence",
|
|
@@ -29,12 +34,53 @@ export function buildCompletionContract({
|
|
|
29
34
|
}
|
|
30
35
|
|
|
31
36
|
export function acceptanceCriterionRows(charter) {
|
|
32
|
-
|
|
33
|
-
|
|
37
|
+
const statements = stringArray(charter?.acceptance_criteria);
|
|
38
|
+
const ids = criterionIds(
|
|
39
|
+
record(charter?.completion_contract).criterion_ids,
|
|
40
|
+
statements.length,
|
|
41
|
+
);
|
|
42
|
+
return statements.map((statement, index) => ({
|
|
43
|
+
id: ids[index],
|
|
34
44
|
statement,
|
|
35
45
|
}));
|
|
36
46
|
}
|
|
37
47
|
|
|
48
|
+
function criterionIds(value, count) {
|
|
49
|
+
if (value === undefined || value === null) {
|
|
50
|
+
return Array.from({ length: count }, (_, index) => `AC-${index + 1}`);
|
|
51
|
+
}
|
|
52
|
+
if (!Array.isArray(value)) throw new Error("Invalid acceptance criterion IDs");
|
|
53
|
+
const ids = value.map(cleanString);
|
|
54
|
+
if (
|
|
55
|
+
ids.length !== count ||
|
|
56
|
+
new Set(ids).size !== ids.length ||
|
|
57
|
+
ids.some((id) => !criterionIdPattern.test(id))
|
|
58
|
+
) {
|
|
59
|
+
throw new Error("Invalid acceptance criterion IDs");
|
|
60
|
+
}
|
|
61
|
+
return ids;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function normalizeExperience(value, userOutcome) {
|
|
65
|
+
const source = record(value);
|
|
66
|
+
return {
|
|
67
|
+
beneficiary: cleanString(source.beneficiary) || "The requester",
|
|
68
|
+
situation:
|
|
69
|
+
cleanString(source.situation) || "Using the requested changed behavior",
|
|
70
|
+
goal: cleanString(source.goal) || userOutcome,
|
|
71
|
+
usable_outcome: cleanString(source.usable_outcome) || userOutcome,
|
|
72
|
+
recovery:
|
|
73
|
+
cleanString(source.recovery) ||
|
|
74
|
+
"Unmet acceptance rows remain visible with the next repair.",
|
|
75
|
+
forbidden_experiences:
|
|
76
|
+
stringArray(source.forbidden_experiences).length > 0
|
|
77
|
+
? stringArray(source.forbidden_experiences)
|
|
78
|
+
: [
|
|
79
|
+
"Reporting completion from components or commands without the end-to-end outcome",
|
|
80
|
+
],
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
38
84
|
export function normalizeUncertaintyVerification(value) {
|
|
39
85
|
const source = record(value);
|
|
40
86
|
return {
|
|
@@ -76,7 +122,7 @@ export function normalizeVisualReview(value) {
|
|
|
76
122
|
|
|
77
123
|
export function completionEvidenceChecks(charter, attempt) {
|
|
78
124
|
const contract = record(charter?.completion_contract);
|
|
79
|
-
if (contract.version !== COMPLETION_CONTRACT_VERSION) return [];
|
|
125
|
+
if (contract.version !== COMPLETION_CONTRACT_VERSION) return [check("completion_contract_version", "Supported completion contract required", false, "Missing or unsupported completion contract; re-charter and verify before promotion.")];
|
|
80
126
|
|
|
81
127
|
const uncertainty = normalizeUncertaintyVerification(attempt?.uncertainty_verification);
|
|
82
128
|
const rows = acceptanceCriterionRows(charter);
|
|
@@ -65,6 +65,7 @@ export function createCodingLoopSnapshot({ runId, request, metadata = {} }) {
|
|
|
65
65
|
const inheritedArtifacts = stringArray(charterContext.artifacts_required);
|
|
66
66
|
const inheritedAssumptions = stringArray(charterContext.assumptions);
|
|
67
67
|
const inheritedCompletionContract = record(charterContext.completion_contract);
|
|
68
|
+
const inheritedExperienceContract = record(charterContext.experience_contract);
|
|
68
69
|
const finalAcceptanceCriteria =
|
|
69
70
|
acceptanceCriteria.length > 0
|
|
70
71
|
? acceptanceCriteria
|
|
@@ -157,7 +158,17 @@ export function createCodingLoopSnapshot({ runId, request, metadata = {} }) {
|
|
|
157
158
|
visual_review_required: browserEvidenceRequired,
|
|
158
159
|
},
|
|
159
160
|
completion_contract: buildCompletionContract({
|
|
160
|
-
inherited:
|
|
161
|
+
inherited: {
|
|
162
|
+
...inheritedCompletionContract,
|
|
163
|
+
...(Object.keys(inheritedExperienceContract).length > 0
|
|
164
|
+
? {
|
|
165
|
+
experience: inheritedExperienceContract,
|
|
166
|
+
criterion_ids:
|
|
167
|
+
inheritedExperienceContract.criterion_ids ??
|
|
168
|
+
inheritedCompletionContract.criterion_ids,
|
|
169
|
+
}
|
|
170
|
+
: {}),
|
|
171
|
+
},
|
|
161
172
|
requirement: cleanString(charterContext.user_goal) || cleanString(request.requirement),
|
|
162
173
|
acceptanceCriteria: finalAcceptanceCriteria,
|
|
163
174
|
nonGoals: finalNonGoals,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { collectMinimumCompletion } from "./minimum-completion.js";
|
|
1
2
|
import { execFile } from "node:child_process";
|
|
2
3
|
import { promisify } from "node:util";
|
|
3
4
|
|
|
@@ -25,9 +26,16 @@ export async function collectPrEvidence({
|
|
|
25
26
|
runCommand,
|
|
26
27
|
});
|
|
27
28
|
const github = await collectGitHubEvidence({ charter, attempt, repositoryPath, runCommand });
|
|
29
|
+
const minimumCompletion = await collectMinimumCompletion({ charter, attempt, repositoryPath, local });
|
|
30
|
+
const latest = await collectLocalGitEvidence({ baseBranch: charter.base_branch, repositoryPath, runCommand });
|
|
31
|
+
if (!latest.available || latest.head_sha !== local.head_sha || !latest.working_tree_clean) {
|
|
32
|
+
minimumCompletion.status = "evidence_incomplete";
|
|
33
|
+
minimumCompletion.errors = ["Checkout changed while collecting minimum evidence; verify again."];
|
|
34
|
+
}
|
|
28
35
|
return {
|
|
36
|
+
minimum_completion: minimumCompletion,
|
|
29
37
|
collected_at: new Date().toISOString(),
|
|
30
|
-
local,
|
|
38
|
+
local: latest,
|
|
31
39
|
github,
|
|
32
40
|
};
|
|
33
41
|
}
|
|
@@ -26,7 +26,9 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
26
26
|
"audit_target",
|
|
27
27
|
[
|
|
28
28
|
`Requirement: ${charter.requirement}`,
|
|
29
|
-
`
|
|
29
|
+
`Original request: ${charter.thread_state?.original_request || charter.requirement}`,
|
|
30
|
+
`Acceptance criteria:\n${bullets(acceptanceCriterionRows(charter).map(row => `${row.id}: ${row.statement}`))}`,
|
|
31
|
+
`Frozen experience (controller-owned): ${JSON.stringify(charter.completion_contract?.experience ?? null)}`,
|
|
30
32
|
`User-visible outcome: ${charter.completion_contract?.user_outcome || charter.requirement}`,
|
|
31
33
|
`Minimum scope:\n${bullets(charter.completion_contract?.minimum_scope || charter.acceptance_criteria)}`,
|
|
32
34
|
`Risk level: ${charter.risk.level}`,
|
|
@@ -43,6 +45,8 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
43
45
|
`Reported tests:\n${bullets(attempt.tests_run || [])}`,
|
|
44
46
|
`Executor uncertainty verification: ${JSON.stringify(attempt.uncertainty_verification || {})}`,
|
|
45
47
|
`Executor criterion evidence: ${JSON.stringify(attempt.criterion_evidence || [])}`,
|
|
48
|
+
`Minimum evidence matrix: ${attempt.observed_evidence?.minimum_completion?.evidence_path || "missing"}`,
|
|
49
|
+
`Minimum evidence digest: ${attempt.observed_evidence?.minimum_completion?.matrix_sha256 || "missing"}`,
|
|
46
50
|
`Executor visual review: ${JSON.stringify(attempt.visual_review || {})}`,
|
|
47
51
|
`Observed local branch: ${localEvidence.branch_name || "unknown"}`,
|
|
48
52
|
`Observed local HEAD: ${localEvidence.head_sha || "unknown"}`,
|
|
@@ -59,7 +63,8 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
59
63
|
"Inspect the final diff and related code directly from the repository.",
|
|
60
64
|
"Look for unmet acceptance criteria, regressions, security issues, data and concurrency risks, missing tests, and unrelated changes.",
|
|
61
65
|
"Treat executor claims as untrusted until supported by repository evidence.",
|
|
62
|
-
"
|
|
66
|
+
"Compare the original request and frozen experience to the complete stable criterion-ID list and implementation mapping. Report forgotten features, dead controls, unsaved results and broken integration. Open the raw minimum evidence, verify its oracle and actual runtime observations; matching expected/actual JSON and hashes alone do not prove truth. Check reload/session return and persisted values/ownership when saving is required.",
|
|
67
|
+
"For every stable criterion ID, return a criterion_results entry only after directly checking its cited proof and observed outcome.",
|
|
63
68
|
charter.evidence_policy?.browser_required
|
|
64
69
|
? "Open and inspect the actual local screenshots or videos. Return visual_review.inspected=true only after checking their visible content against the acceptance criteria."
|
|
65
70
|
: "Do not require visual evidence when the Charter does not require browser evidence.",
|
|
@@ -70,9 +75,10 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
70
75
|
"result_contract",
|
|
71
76
|
[
|
|
72
77
|
"Finish with exactly one line beginning SIMY_AUDIT_JSON: followed by one JSON object.",
|
|
73
|
-
"Required keys: passed, summary, findings, criterion_results, visual_review.",
|
|
78
|
+
"Required keys: passed, summary, findings, criterion_results, visual_review, coverage_review.",
|
|
79
|
+
"coverage_review must contain requirements_complete (boolean), unmapped_requirements (array), and observation explaining the comparison of the original request, frozen experience, stable criterion-ID list, implementation and raw user/system proof. Use false and list gaps when anything is missing or unverified.",
|
|
74
80
|
"Each finding candidate must contain passed, code, severity, target, explanation, and repairability.",
|
|
75
|
-
"criterion_results must contain one entry per
|
|
81
|
+
"criterion_results must contain one entry per stable criterion ID with criterion_id, passed, verification_method, evidence_refs, and observed_result.",
|
|
76
82
|
"visual_review must contain inspected, artifacts, and acceptance_observations.",
|
|
77
83
|
"Only passed=false entries are findings. Omit passing checks, and never report passed=true evidence as a finding.",
|
|
78
84
|
"Use passed=false when a blocker or major finding remains, or when evidence is unavailable.",
|
|
@@ -195,12 +201,19 @@ export function buildIndependentAudit(execution, charter = null, attempt = null)
|
|
|
195
201
|
auto_fix_hint: null,
|
|
196
202
|
});
|
|
197
203
|
}
|
|
204
|
+
const coverage = result.coverage_review;
|
|
205
|
+
if (attempt && !(coverage?.requirements_complete === true &&
|
|
206
|
+
Array.isArray(coverage.unmapped_requirements) && coverage.unmapped_requirements.length === 0 && cleanString(coverage.observation))) {
|
|
207
|
+
findings.push({ passed: false, code: "MINIMUM_REQUIREMENT_COVERAGE_UNPROVEN", severity: "blocker",
|
|
208
|
+
target: "coverage_review", explanation: "Original request coverage and raw minimum evidence were not independently verified.", repairability: "auto" });
|
|
209
|
+
}
|
|
198
210
|
const requiresHuman = findings.some((item) => item.repairability === "manual");
|
|
199
211
|
return {
|
|
200
212
|
passed: result.passed === true && findings.length === 0,
|
|
201
213
|
summary: cleanString(result.summary) || "Independent audit completed.",
|
|
202
214
|
findings,
|
|
203
215
|
criterion_results: criterionResults,
|
|
216
|
+
coverage_review: coverage || null,
|
|
204
217
|
visual_review: visualReview,
|
|
205
218
|
requires_human: requiresHuman,
|
|
206
219
|
};
|