@awak-app/simy-cli 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/cli-contract.js +2 -2
- package/src/orchestrator/audit.js +90 -2
- package/src/orchestrator/completion-contract.js +159 -0
- package/src/orchestrator/contract.js +23 -12
- package/src/orchestrator/independent-audit.js +88 -3
- package/src/orchestrator/instruction.js +26 -4
- package/src/orchestrator/loop.js +5 -1
- package/src/orchestrator/result.js +10 -0
- package/src/runner.js +1 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@awak-app/simy-cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Local SIMY Agentic Loop executor for Codex and Claude Code.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -26,12 +26,13 @@
|
|
|
26
26
|
"test:e2e:budgets": "node ./scripts/budget-enforcement-e2e.js",
|
|
27
27
|
"test:e2e:desktop-executor": "node ./scripts/desktop-executor-e2e.js",
|
|
28
28
|
"test:e2e:pre-guardrail": "node ./scripts/pre-guardrail-e2e.js",
|
|
29
|
+
"test:e2e:completion-contract": "node ./scripts/completion-contract-e2e.js",
|
|
29
30
|
"test:e2e:console:real-provider": "node ./scripts/real-provider-console-e2e.js",
|
|
30
31
|
"test:e2e:console:fullstack": "node ./scripts/fullstack-cli-smoke.js",
|
|
31
32
|
"check": "npm run check:auto-update && node --check ./src/desktop-executor.js && node --check ./scripts/desktop-executor-e2e.js && node --check ./scripts/pre-guardrail-e2e.js && node --check ./scripts/fixtures/fake-desktop-provider.js && npm run check:syntax && npm run check:real-provider-syntax && npm test && npm run check:package",
|
|
32
33
|
"check:auto-update": "node --check ./src/auto-update.js && node --check ./scripts/auto-update-e2e.js",
|
|
33
34
|
"check:real-provider-syntax": "node --check ./scripts/real-codex-probe.js && node --check ./scripts/real-claude-probe.js && node --check ./scripts/real-provider-console-fixture.js && node --check ./scripts/real-provider-console-e2e.js",
|
|
34
|
-
"check:syntax": "node --check ./src/index.js && node --check ./src/agent.js && node --check ./src/browser.js && node --check ./src/web-api.js && node --check ./src/run-registry.js && node --check ./src/direct-executor.js && node --check ./src/execution-guardrail.js && node --check ./src/backend-executable.js && node --check ./src/provider-stream.js && node --check ./src/console/index.js && node --check ./src/console/app.js && node --check ./src/console/commands.js && node --check ./src/local-attachments.js && node --check ./src/repository-inventory.js && node --check ./src/session-store.js && node --check ./src/web-origin.js && node --check ./src/workspace-context.js && node --check ./src/orchestrator/index.js && node --check ./src/orchestrator/shared.js && node --check ./src/orchestrator/recovery.js && node --check ./src/orchestrator/risk.js && node --check ./src/orchestrator/contract.js && node --check ./src/orchestrator/instruction.js && node --check ./src/orchestrator/problem-solving.js && node --check ./src/orchestrator/budget.js && node --check ./src/orchestrator/execution-io.js && node --check ./src/orchestrator/presentation.js && node --check ./src/orchestrator/result.js && node --check ./src/orchestrator/retry.js && node --check ./src/orchestrator/evidence.js && node --check ./src/orchestrator/independent-audit.js && node --check ./src/orchestrator/audit.js && node --check ./src/orchestrator/loop.js && node --check ./src/runner.js && node --check ./scripts/fixtures/fake-coding-backend.js && node --check ./scripts/e2e-terminal-evidence.js && node --check ./scripts/audit-gate-e2e.js && node --check ./scripts/retry-circuit-e2e.js && node --check ./scripts/follow-up-charter-e2e.js && node --check ./scripts/task-routing-e2e.js && node --check ./scripts/executor-version-preflight-e2e.js && node --check ./scripts/blocked-recovery-e2e.js && node --check ./scripts/process-localization-e2e.js && node --check ./scripts/hypothesis-retries-e2e.js && node --check ./scripts/budget-enforcement-e2e.js && node --check ./scripts/console-e2e-fixture.js && node --check ./scripts/console-e2e.js && node --check ./scripts/real-provider-console-fixture.js && node --check ./scripts/real-provider-console-e2e.js && node --check ./scripts/fullstack-cli-smoke.js && node --check ./scripts/check-package-contents.js",
|
|
35
|
+
"check:syntax": "node --check ./src/index.js && node --check ./src/agent.js && node --check ./src/browser.js && node --check ./src/web-api.js && node --check ./src/run-registry.js && node --check ./src/direct-executor.js && node --check ./src/execution-guardrail.js && node --check ./src/backend-executable.js && node --check ./src/provider-stream.js && node --check ./src/console/index.js && node --check ./src/console/app.js && node --check ./src/console/commands.js && node --check ./src/local-attachments.js && node --check ./src/repository-inventory.js && node --check ./src/session-store.js && node --check ./src/web-origin.js && node --check ./src/workspace-context.js && node --check ./src/orchestrator/index.js && node --check ./src/orchestrator/shared.js && node --check ./src/orchestrator/recovery.js && node --check ./src/orchestrator/risk.js && node --check ./src/orchestrator/contract.js && node --check ./src/orchestrator/completion-contract.js && node --check ./src/orchestrator/instruction.js && node --check ./src/orchestrator/problem-solving.js && node --check ./src/orchestrator/budget.js && node --check ./src/orchestrator/execution-io.js && node --check ./src/orchestrator/presentation.js && node --check ./src/orchestrator/result.js && node --check ./src/orchestrator/retry.js && node --check ./src/orchestrator/evidence.js && node --check ./src/orchestrator/independent-audit.js && node --check ./src/orchestrator/audit.js && node --check ./src/orchestrator/loop.js && node --check ./src/runner.js && node --check ./scripts/fixtures/completion-contract-fixture.js && node --check ./scripts/fixtures/fake-coding-backend.js && node --check ./scripts/e2e-terminal-evidence.js && node --check ./scripts/audit-gate-e2e.js && node --check ./scripts/completion-contract-e2e.js && node --check ./scripts/retry-circuit-e2e.js && node --check ./scripts/follow-up-charter-e2e.js && node --check ./scripts/task-routing-e2e.js && node --check ./scripts/executor-version-preflight-e2e.js && node --check ./scripts/blocked-recovery-e2e.js && node --check ./scripts/process-localization-e2e.js && node --check ./scripts/hypothesis-retries-e2e.js && node --check ./scripts/budget-enforcement-e2e.js && node --check ./scripts/console-e2e-fixture.js && node --check ./scripts/console-e2e.js && node --check ./scripts/real-provider-console-fixture.js && node --check ./scripts/real-provider-console-e2e.js && node --check ./scripts/fullstack-cli-smoke.js && node --check ./scripts/check-package-contents.js",
|
|
35
36
|
"check:package": "node ./scripts/check-package-contents.js"
|
|
36
37
|
},
|
|
37
38
|
"engines": {
|
package/src/cli-contract.js
CHANGED
|
@@ -8,11 +8,23 @@ import {
|
|
|
8
8
|
stringArray,
|
|
9
9
|
} from "./shared.js";
|
|
10
10
|
import { evaluateRetryProblemSolving } from "./problem-solving.js";
|
|
11
|
+
import {
|
|
12
|
+
completionEvidenceChecks,
|
|
13
|
+
normalizeVisualReview,
|
|
14
|
+
} from "./completion-contract.js";
|
|
11
15
|
|
|
12
16
|
export async function auditAttempt(charter, attempt, { previousAttempt = null } = {}) {
|
|
13
17
|
const checks = [];
|
|
14
|
-
const addCheck = (
|
|
15
|
-
|
|
18
|
+
const addCheck = (
|
|
19
|
+
id,
|
|
20
|
+
label,
|
|
21
|
+
passed,
|
|
22
|
+
detail,
|
|
23
|
+
severity = "blocker",
|
|
24
|
+
repairability = "auto",
|
|
25
|
+
evidenceRefs = [],
|
|
26
|
+
) => {
|
|
27
|
+
checks.push({ id, label, passed, severity, detail, repairability, evidence_refs: evidenceRefs });
|
|
16
28
|
};
|
|
17
29
|
const local = attempt.observed_evidence?.local || {};
|
|
18
30
|
|
|
@@ -119,6 +131,18 @@ export async function auditAttempt(charter, attempt, { previousAttempt = null }
|
|
|
119
131
|
}
|
|
120
132
|
attempt.retry_problem_solving = retryProblemSolving;
|
|
121
133
|
|
|
134
|
+
for (const check of completionEvidenceChecks(charter, attempt)) {
|
|
135
|
+
addCheck(
|
|
136
|
+
check.id,
|
|
137
|
+
check.label,
|
|
138
|
+
check.passed,
|
|
139
|
+
check.detail,
|
|
140
|
+
"blocker",
|
|
141
|
+
"auto",
|
|
142
|
+
check.evidence_refs,
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
122
146
|
const evidenceRequired = charter.expected_evidence.includes("ui_evidence_path");
|
|
123
147
|
const evidenceArtifact = await inspectEvidence(charter, attempt.ui_evidence_path);
|
|
124
148
|
if (evidenceRequired || attempt.ui_evidence_path) {
|
|
@@ -142,6 +166,32 @@ export async function auditAttempt(charter, attempt, { previousAttempt = null }
|
|
|
142
166
|
? `${evidenceArtifact.visual_file_count} screenshot or video artifact(s) found.`
|
|
143
167
|
: "UI and browser-extension work requires a real screenshot or video; a path, text file, or empty directory is not sufficient.",
|
|
144
168
|
);
|
|
169
|
+
const visualReview = normalizeVisualReview(attempt.visual_review);
|
|
170
|
+
const visualArtifactReview = await inspectVisualReviewArtifacts(charter, visualReview);
|
|
171
|
+
addCheck(
|
|
172
|
+
"visual_content_inspected",
|
|
173
|
+
"Visual evidence content was inspected against acceptance criteria",
|
|
174
|
+
Boolean(
|
|
175
|
+
visualReview.inspected &&
|
|
176
|
+
visualReview.artifacts.length > 0 &&
|
|
177
|
+
visualReview.acceptance_observations.length > 0,
|
|
178
|
+
),
|
|
179
|
+
visualReview.inspected && visualReview.acceptance_observations.length > 0
|
|
180
|
+
? visualReview.acceptance_observations.join("; ")
|
|
181
|
+
: "UI evidence must be opened and inspected; record the artifact paths and concrete acceptance observations.",
|
|
182
|
+
"blocker",
|
|
183
|
+
"auto",
|
|
184
|
+
visualReview.artifacts,
|
|
185
|
+
);
|
|
186
|
+
addCheck(
|
|
187
|
+
"visual_review_artifacts_verified",
|
|
188
|
+
"Claimed visual-review artifacts are real local media",
|
|
189
|
+
visualArtifactReview.valid,
|
|
190
|
+
visualArtifactReview.summary,
|
|
191
|
+
"blocker",
|
|
192
|
+
"auto",
|
|
193
|
+
visualArtifactReview.verified,
|
|
194
|
+
);
|
|
145
195
|
}
|
|
146
196
|
|
|
147
197
|
const findings = checks
|
|
@@ -409,3 +459,41 @@ async function isVisualEvidenceFile(filePath) {
|
|
|
409
459
|
header.subarray(0, 4).equals(Buffer.from([26, 69, 223, 163]))
|
|
410
460
|
);
|
|
411
461
|
}
|
|
462
|
+
|
|
463
|
+
async function inspectVisualReviewArtifacts(charter, visualReview) {
|
|
464
|
+
const root = charter.ui_evidence_root ? path.resolve(charter.ui_evidence_root) : null;
|
|
465
|
+
const verified = [];
|
|
466
|
+
const invalid = [];
|
|
467
|
+
for (const artifact of visualReview.artifacts) {
|
|
468
|
+
const resolved = path.resolve(artifact);
|
|
469
|
+
const underRoot = Boolean(
|
|
470
|
+
root && (resolved === root || resolved.startsWith(`${root}${path.sep}`)),
|
|
471
|
+
);
|
|
472
|
+
let fileStat = null;
|
|
473
|
+
try {
|
|
474
|
+
fileStat = await stat(resolved);
|
|
475
|
+
} catch {
|
|
476
|
+
// The invalid reason below covers missing files.
|
|
477
|
+
}
|
|
478
|
+
if (
|
|
479
|
+
!underRoot ||
|
|
480
|
+
!fileStat?.isFile() ||
|
|
481
|
+
fileStat.size <= 0 ||
|
|
482
|
+
!(await isVisualEvidenceFile(resolved))
|
|
483
|
+
) {
|
|
484
|
+
invalid.push(artifact);
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
verified.push(resolved);
|
|
488
|
+
}
|
|
489
|
+
return {
|
|
490
|
+
valid: visualReview.artifacts.length > 0 && invalid.length === 0,
|
|
491
|
+
verified,
|
|
492
|
+
summary:
|
|
493
|
+
visualReview.artifacts.length === 0
|
|
494
|
+
? "No inspected visual artifact paths were recorded."
|
|
495
|
+
: invalid.length > 0
|
|
496
|
+
? `Missing, non-visual, or out-of-root artifacts: ${invalid.join(", ")}`
|
|
497
|
+
: `${verified.length} inspected visual artifact(s) exist under the required local root with valid media signatures.`,
|
|
498
|
+
};
|
|
499
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { cleanString, stringArray } from "./shared.js";
|
|
2
|
+
|
|
3
|
+
export const COMPLETION_CONTRACT_VERSION = 2;
|
|
4
|
+
|
|
5
|
+
export function buildCompletionContract({
|
|
6
|
+
inherited,
|
|
7
|
+
requirement,
|
|
8
|
+
acceptanceCriteria,
|
|
9
|
+
nonGoals,
|
|
10
|
+
expectedEvidence,
|
|
11
|
+
}) {
|
|
12
|
+
const source = record(inherited);
|
|
13
|
+
const minimumScope = stringArray(source.minimum_scope);
|
|
14
|
+
const proofRequired = stringArray(source.proof_required);
|
|
15
|
+
return {
|
|
16
|
+
version: COMPLETION_CONTRACT_VERSION,
|
|
17
|
+
user_outcome: cleanString(source.user_outcome) || cleanString(requirement),
|
|
18
|
+
minimum_scope:
|
|
19
|
+
minimumScope.length > 0 ? minimumScope : stringArray(acceptanceCriteria),
|
|
20
|
+
non_goals: stringArray(nonGoals),
|
|
21
|
+
proof_required: uniqueStrings([
|
|
22
|
+
"criterion_evidence",
|
|
23
|
+
...proofRequired,
|
|
24
|
+
...stringArray(expectedEvidence),
|
|
25
|
+
]),
|
|
26
|
+
deadline: cleanString(source.deadline) || null,
|
|
27
|
+
milestones: stringArray(source.milestones),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function acceptanceCriterionRows(charter) {
|
|
32
|
+
return stringArray(charter?.acceptance_criteria).map((statement, index) => ({
|
|
33
|
+
id: `AC-${index + 1}`,
|
|
34
|
+
statement,
|
|
35
|
+
}));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function normalizeUncertaintyVerification(value) {
|
|
39
|
+
const source = record(value);
|
|
40
|
+
return {
|
|
41
|
+
top_uncertainty: cleanString(source.top_uncertainty) || null,
|
|
42
|
+
first_verification: cleanString(source.first_verification) || null,
|
|
43
|
+
evidence_checked: uniqueStrings(source.evidence_checked),
|
|
44
|
+
observed_result: cleanString(source.observed_result) || null,
|
|
45
|
+
resolved: source.resolved === true,
|
|
46
|
+
fallback: cleanString(source.fallback) || null,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function normalizeCriterionEvidence(value) {
|
|
51
|
+
if (!Array.isArray(value)) return [];
|
|
52
|
+
return value
|
|
53
|
+
.map((item) => {
|
|
54
|
+
const source = record(item);
|
|
55
|
+
const criterionId = cleanString(source.criterion_id);
|
|
56
|
+
if (!criterionId) return null;
|
|
57
|
+
return {
|
|
58
|
+
criterion_id: criterionId,
|
|
59
|
+
passed: source.passed === true,
|
|
60
|
+
verification_method: cleanString(source.verification_method) || null,
|
|
61
|
+
evidence_refs: uniqueStrings(source.evidence_refs),
|
|
62
|
+
observed_result: cleanString(source.observed_result) || null,
|
|
63
|
+
};
|
|
64
|
+
})
|
|
65
|
+
.filter(Boolean);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function normalizeVisualReview(value) {
|
|
69
|
+
const source = record(value);
|
|
70
|
+
return {
|
|
71
|
+
inspected: source.inspected === true,
|
|
72
|
+
artifacts: uniqueStrings(source.artifacts),
|
|
73
|
+
acceptance_observations: uniqueStrings(source.acceptance_observations),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function completionEvidenceChecks(charter, attempt) {
|
|
78
|
+
const contract = record(charter?.completion_contract);
|
|
79
|
+
if (contract.version !== COMPLETION_CONTRACT_VERSION) return [];
|
|
80
|
+
|
|
81
|
+
const uncertainty = normalizeUncertaintyVerification(attempt?.uncertainty_verification);
|
|
82
|
+
const criterionEvidence = normalizeCriterionEvidence(attempt?.criterion_evidence);
|
|
83
|
+
const rows = acceptanceCriterionRows(charter);
|
|
84
|
+
const checks = [
|
|
85
|
+
check(
|
|
86
|
+
"top_uncertainty_recorded",
|
|
87
|
+
"Highest uncertainty is explicit",
|
|
88
|
+
Boolean(uncertainty.top_uncertainty),
|
|
89
|
+
uncertainty.top_uncertainty || "No highest uncertainty was recorded.",
|
|
90
|
+
),
|
|
91
|
+
check(
|
|
92
|
+
"first_verification_recorded",
|
|
93
|
+
"First discriminating verification is explicit",
|
|
94
|
+
Boolean(uncertainty.first_verification),
|
|
95
|
+
uncertainty.first_verification || "No first verification was recorded.",
|
|
96
|
+
),
|
|
97
|
+
check(
|
|
98
|
+
"uncertainty_evidence_recorded",
|
|
99
|
+
"Uncertainty verification records concrete evidence",
|
|
100
|
+
uncertainty.evidence_checked.length > 0 && Boolean(uncertainty.observed_result),
|
|
101
|
+
uncertainty.evidence_checked.length > 0 && uncertainty.observed_result
|
|
102
|
+
? `${uncertainty.evidence_checked.length} evidence item(s): ${uncertainty.observed_result}`
|
|
103
|
+
: "The uncertainty check needs evidence_checked and an observed_result.",
|
|
104
|
+
),
|
|
105
|
+
check(
|
|
106
|
+
"top_uncertainty_resolved",
|
|
107
|
+
"Highest uncertainty is resolved before completion",
|
|
108
|
+
uncertainty.resolved,
|
|
109
|
+
uncertainty.resolved
|
|
110
|
+
? "The executor marked the highest uncertainty resolved with observed evidence."
|
|
111
|
+
: uncertainty.fallback || "The highest uncertainty remains unresolved.",
|
|
112
|
+
),
|
|
113
|
+
];
|
|
114
|
+
|
|
115
|
+
for (const row of rows) {
|
|
116
|
+
const matches = criterionEvidence.filter((item) => item.criterion_id === row.id);
|
|
117
|
+
const evidence = matches[0];
|
|
118
|
+
checks.push(
|
|
119
|
+
check(
|
|
120
|
+
`criterion_evidence_${row.id.toLowerCase()}`,
|
|
121
|
+
`${row.id} has passing completion evidence`,
|
|
122
|
+
Boolean(
|
|
123
|
+
matches.length === 1 &&
|
|
124
|
+
evidence?.passed &&
|
|
125
|
+
evidence.verification_method &&
|
|
126
|
+
evidence.evidence_refs.length > 0 &&
|
|
127
|
+
evidence.observed_result,
|
|
128
|
+
),
|
|
129
|
+
matches.length > 1
|
|
130
|
+
? `Expected exactly one evidence result for ${row.id}; recorded ${matches.length}.`
|
|
131
|
+
: evidence
|
|
132
|
+
? `${evidence.verification_method || "method missing"}: ${
|
|
133
|
+
evidence.observed_result || "observed result missing"
|
|
134
|
+
}; evidence ${evidence.evidence_refs.join(", ") || "missing"}.`
|
|
135
|
+
: `No criterion evidence was recorded for ${row.id}: ${row.statement}`,
|
|
136
|
+
evidence?.evidence_refs || [],
|
|
137
|
+
),
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
return checks;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function check(id, label, passed, detail, evidenceRefs = []) {
|
|
144
|
+
return {
|
|
145
|
+
id,
|
|
146
|
+
label,
|
|
147
|
+
passed: Boolean(passed),
|
|
148
|
+
detail,
|
|
149
|
+
evidence_refs: evidenceRefs,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function record(value) {
|
|
154
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function uniqueStrings(value) {
|
|
158
|
+
return [...new Set(stringArray(value))];
|
|
159
|
+
}
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from "./shared.js";
|
|
10
10
|
import { budgetState, resolveRunBudgets } from "./budget.js";
|
|
11
11
|
import { classifyRisk } from "./risk.js";
|
|
12
|
+
import { buildCompletionContract } from "./completion-contract.js";
|
|
12
13
|
import {
|
|
13
14
|
DESKTOP_EXECUTION_TARGET,
|
|
14
15
|
normalizeDesktopExecutionTarget,
|
|
@@ -63,6 +64,19 @@ export function createCodingLoopSnapshot({ runId, request, metadata = {} }) {
|
|
|
63
64
|
const inheritedNonGoals = stringArray(charterContext.non_goals);
|
|
64
65
|
const inheritedArtifacts = stringArray(charterContext.artifacts_required);
|
|
65
66
|
const inheritedAssumptions = stringArray(charterContext.assumptions);
|
|
67
|
+
const inheritedCompletionContract = record(charterContext.completion_contract);
|
|
68
|
+
const finalAcceptanceCriteria =
|
|
69
|
+
acceptanceCriteria.length > 0
|
|
70
|
+
? acceptanceCriteria
|
|
71
|
+
: [
|
|
72
|
+
"The requested change is implemented without unrelated scope.",
|
|
73
|
+
"Relevant tests pass and are recorded.",
|
|
74
|
+
`A pull request targets ${baseBranch}.`,
|
|
75
|
+
];
|
|
76
|
+
const finalNonGoals =
|
|
77
|
+
inheritedNonGoals.length > 0
|
|
78
|
+
? inheritedNonGoals
|
|
79
|
+
: ["Unrelated refactors", "Server-side execution of repository code"];
|
|
66
80
|
const designReviewRequired =
|
|
67
81
|
risk.requires_design_review || inheritedDesignReview.required === true;
|
|
68
82
|
const designReviewApproved =
|
|
@@ -101,14 +115,7 @@ export function createCodingLoopSnapshot({ runId, request, metadata = {} }) {
|
|
|
101
115
|
retry_budget: budgets.retry_budget,
|
|
102
116
|
token_budget: budgets.token_budget,
|
|
103
117
|
ui_evidence_root: cleanString(request.ui_evidence_root),
|
|
104
|
-
acceptance_criteria:
|
|
105
|
-
acceptanceCriteria.length > 0
|
|
106
|
-
? acceptanceCriteria
|
|
107
|
-
: [
|
|
108
|
-
"The requested change is implemented without unrelated scope.",
|
|
109
|
-
"Relevant tests pass and are recorded.",
|
|
110
|
-
`A pull request targets ${baseBranch}.`,
|
|
111
|
-
],
|
|
118
|
+
acceptance_criteria: finalAcceptanceCriteria,
|
|
112
119
|
acceptance_criteria_source: acceptanceCriteriaSource,
|
|
113
120
|
expected_tests: stringArray(request.expected_tests),
|
|
114
121
|
expected_evidence: expectedEvidence,
|
|
@@ -149,14 +156,18 @@ export function createCodingLoopSnapshot({ runId, request, metadata = {} }) {
|
|
|
149
156
|
accepted_artifact_kinds: browserEvidenceRequired ? ["image", "video"] : [],
|
|
150
157
|
visual_review_required: browserEvidenceRequired,
|
|
151
158
|
},
|
|
159
|
+
completion_contract: buildCompletionContract({
|
|
160
|
+
inherited: inheritedCompletionContract,
|
|
161
|
+
requirement: cleanString(charterContext.user_goal) || cleanString(request.requirement),
|
|
162
|
+
acceptanceCriteria: finalAcceptanceCriteria,
|
|
163
|
+
nonGoals: finalNonGoals,
|
|
164
|
+
expectedEvidence,
|
|
165
|
+
}),
|
|
152
166
|
thread_state: {
|
|
153
167
|
original_request:
|
|
154
168
|
cleanString(charterContext.original_request) || cleanString(request.requirement),
|
|
155
169
|
user_goal: cleanString(charterContext.user_goal) || cleanString(request.requirement),
|
|
156
|
-
non_goals:
|
|
157
|
-
inheritedNonGoals.length > 0
|
|
158
|
-
? inheritedNonGoals
|
|
159
|
-
: ["Unrelated refactors", "Server-side execution of repository code"],
|
|
170
|
+
non_goals: finalNonGoals,
|
|
160
171
|
expected_finish_line:
|
|
161
172
|
cleanString(charterContext.expected_finish_line) ||
|
|
162
173
|
`A verified pull request targeting ${baseBranch}.`,
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { bullets, cleanString, section } from "./shared.js";
|
|
2
|
+
import {
|
|
3
|
+
COMPLETION_CONTRACT_VERSION,
|
|
4
|
+
acceptanceCriterionRows,
|
|
5
|
+
normalizeCriterionEvidence,
|
|
6
|
+
normalizeVisualReview,
|
|
7
|
+
} from "./completion-contract.js";
|
|
2
8
|
|
|
3
9
|
export function buildIndependentAuditInstruction(charter, attempt) {
|
|
4
10
|
const localEvidence = attempt.observed_evidence?.local || {};
|
|
@@ -21,6 +27,8 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
21
27
|
[
|
|
22
28
|
`Requirement: ${charter.requirement}`,
|
|
23
29
|
`Acceptance criteria:\n${bullets(charter.acceptance_criteria)}`,
|
|
30
|
+
`User-visible outcome: ${charter.completion_contract?.user_outcome || charter.requirement}`,
|
|
31
|
+
`Minimum scope:\n${bullets(charter.completion_contract?.minimum_scope || charter.acceptance_criteria)}`,
|
|
24
32
|
`Risk level: ${charter.risk.level}`,
|
|
25
33
|
`Risk tags: ${charter.risk.tags.join(", ") || "none"}`,
|
|
26
34
|
`Base branch: ${charter.base_branch}`,
|
|
@@ -33,6 +41,9 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
33
41
|
`Local UI evidence path: ${attempt.ui_evidence_path || "not reported"}`,
|
|
34
42
|
`Local evidence verification: ${evidenceArtifact?.summary || "not available"}`,
|
|
35
43
|
`Reported tests:\n${bullets(attempt.tests_run || [])}`,
|
|
44
|
+
`Executor uncertainty verification: ${JSON.stringify(attempt.uncertainty_verification || {})}`,
|
|
45
|
+
`Executor criterion evidence: ${JSON.stringify(attempt.criterion_evidence || [])}`,
|
|
46
|
+
`Executor visual review: ${JSON.stringify(attempt.visual_review || {})}`,
|
|
36
47
|
`Observed local branch: ${localEvidence.branch_name || "unknown"}`,
|
|
37
48
|
`Observed local HEAD: ${localEvidence.head_sha || "unknown"}`,
|
|
38
49
|
`Observed changed files:\n${bullets(localEvidence.changed_files || [])}`,
|
|
@@ -48,6 +59,10 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
48
59
|
"Inspect the final diff and related code directly from the repository.",
|
|
49
60
|
"Look for unmet acceptance criteria, regressions, security issues, data and concurrency risks, missing tests, and unrelated changes.",
|
|
50
61
|
"Treat executor claims as untrusted until supported by repository evidence.",
|
|
62
|
+
"For every AC-N criterion, return a criterion_results entry only after directly checking its cited proof and observed outcome.",
|
|
63
|
+
charter.evidence_policy?.browser_required
|
|
64
|
+
? "Open and inspect the actual local screenshots or videos. Return visual_review.inspected=true only after checking their visible content against the acceptance criteria."
|
|
65
|
+
: "Do not require visual evidence when the Charter does not require browser evidence.",
|
|
51
66
|
"Do not modify files, commit, push, or update the pull request.",
|
|
52
67
|
].join("\n"),
|
|
53
68
|
),
|
|
@@ -55,8 +70,10 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
55
70
|
"result_contract",
|
|
56
71
|
[
|
|
57
72
|
"Finish with exactly one line beginning SIMY_AUDIT_JSON: followed by one JSON object.",
|
|
58
|
-
"Required keys: passed, summary, findings.",
|
|
73
|
+
"Required keys: passed, summary, findings, criterion_results, visual_review.",
|
|
59
74
|
"Each finding candidate must contain passed, code, severity, target, explanation, and repairability.",
|
|
75
|
+
"criterion_results must contain one entry per AC-N with criterion_id, passed, verification_method, evidence_refs, and observed_result.",
|
|
76
|
+
"visual_review must contain inspected, artifacts, and acceptance_observations.",
|
|
60
77
|
"Only passed=false entries are findings. Omit passing checks, and never report passed=true evidence as a finding.",
|
|
61
78
|
"Use passed=false when a blocker or major finding remains, or when evidence is unavailable.",
|
|
62
79
|
].join("\n"),
|
|
@@ -64,13 +81,19 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
64
81
|
].join("\n\n");
|
|
65
82
|
}
|
|
66
83
|
|
|
67
|
-
export function buildIndependentAudit(execution) {
|
|
84
|
+
export function buildIndependentAudit(execution, charter = null, attempt = null) {
|
|
68
85
|
const result = execution?.result && typeof execution.result === "object" ? execution.result : {};
|
|
86
|
+
const completionContractV2 =
|
|
87
|
+
charter?.completion_contract?.version === COMPLETION_CONTRACT_VERSION;
|
|
69
88
|
const valid =
|
|
70
89
|
execution?.exitCode === 0 &&
|
|
71
90
|
!execution?.error &&
|
|
72
91
|
typeof result.passed === "boolean" &&
|
|
73
|
-
Array.isArray(result.findings)
|
|
92
|
+
Array.isArray(result.findings) &&
|
|
93
|
+
(!completionContractV2 ||
|
|
94
|
+
(Array.isArray(result.criterion_results) &&
|
|
95
|
+
result.visual_review &&
|
|
96
|
+
typeof result.visual_review === "object"));
|
|
74
97
|
if (!valid) {
|
|
75
98
|
return {
|
|
76
99
|
passed: false,
|
|
@@ -85,6 +108,8 @@ export function buildIndependentAudit(execution) {
|
|
|
85
108
|
repairability: "manual",
|
|
86
109
|
},
|
|
87
110
|
],
|
|
111
|
+
criterion_results: [],
|
|
112
|
+
visual_review: normalizeVisualReview(null),
|
|
88
113
|
requires_human: true,
|
|
89
114
|
};
|
|
90
115
|
}
|
|
@@ -101,6 +126,64 @@ export function buildIndependentAudit(execution) {
|
|
|
101
126
|
auto_fix_hint: cleanString(item.auto_fix_hint) || null,
|
|
102
127
|
}));
|
|
103
128
|
const findings = evaluations.filter((item) => item.passed === false);
|
|
129
|
+
const criterionResults = normalizeCriterionEvidence(result.criterion_results);
|
|
130
|
+
if (completionContractV2) {
|
|
131
|
+
for (const criterion of acceptanceCriterionRows(charter)) {
|
|
132
|
+
const matches = criterionResults.filter(
|
|
133
|
+
(item) => item.criterion_id === criterion.id,
|
|
134
|
+
);
|
|
135
|
+
const criterionResult = matches[0];
|
|
136
|
+
if (
|
|
137
|
+
matches.length === 1 &&
|
|
138
|
+
criterionResult?.passed &&
|
|
139
|
+
criterionResult.verification_method &&
|
|
140
|
+
criterionResult.evidence_refs.length > 0 &&
|
|
141
|
+
criterionResult.observed_result
|
|
142
|
+
) {
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
findings.push({
|
|
146
|
+
passed: false,
|
|
147
|
+
code: `INDEPENDENT_${criterion.id.replace("-", "_")}_UNPROVEN`,
|
|
148
|
+
severity: "blocker",
|
|
149
|
+
target: criterion.id,
|
|
150
|
+
explanation: matches.length > 1
|
|
151
|
+
? `The independent result returned ${matches.length} entries for ${criterion.id}; exactly one is required.`
|
|
152
|
+
: criterionResult
|
|
153
|
+
? `The independent result did not positively prove ${criterion.id}: ${criterion.statement}`
|
|
154
|
+
: `The independent result omitted ${criterion.id}: ${criterion.statement}`,
|
|
155
|
+
repairability: "auto",
|
|
156
|
+
auto_fix_hint: "Re-audit this acceptance criterion and cite directly inspected evidence.",
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const visualReview = normalizeVisualReview(result.visual_review);
|
|
161
|
+
const executorVisualReview = normalizeVisualReview(attempt?.visual_review);
|
|
162
|
+
const visualArtifactsOverlap = visualReview.artifacts.some((artifact) =>
|
|
163
|
+
executorVisualReview.artifacts.includes(artifact),
|
|
164
|
+
);
|
|
165
|
+
if (
|
|
166
|
+
completionContractV2 &&
|
|
167
|
+
charter.evidence_policy?.browser_required &&
|
|
168
|
+
!(
|
|
169
|
+
visualReview.inspected &&
|
|
170
|
+
visualReview.artifacts.length > 0 &&
|
|
171
|
+
visualReview.acceptance_observations.length > 0 &&
|
|
172
|
+
(!attempt || visualArtifactsOverlap)
|
|
173
|
+
)
|
|
174
|
+
) {
|
|
175
|
+
findings.push({
|
|
176
|
+
passed: false,
|
|
177
|
+
code: "INDEPENDENT_VISUAL_REVIEW_UNPROVEN",
|
|
178
|
+
severity: "blocker",
|
|
179
|
+
target: "visual_review",
|
|
180
|
+
explanation: attempt && !visualArtifactsOverlap
|
|
181
|
+
? "The independent auditor did not inspect any artifact from the executor's verified visual-review set."
|
|
182
|
+
: "The independent auditor did not record inspection of the actual visual evidence content.",
|
|
183
|
+
repairability: "auto",
|
|
184
|
+
auto_fix_hint: "Open the local screenshots or videos and record acceptance observations.",
|
|
185
|
+
});
|
|
186
|
+
}
|
|
104
187
|
if (result.passed === false && findings.length === 0) {
|
|
105
188
|
findings.push({
|
|
106
189
|
passed: false,
|
|
@@ -117,6 +200,8 @@ export function buildIndependentAudit(execution) {
|
|
|
117
200
|
passed: result.passed === true && findings.length === 0,
|
|
118
201
|
summary: cleanString(result.summary) || "Independent audit completed.",
|
|
119
202
|
findings,
|
|
203
|
+
criterion_results: criterionResults,
|
|
204
|
+
visual_review: visualReview,
|
|
120
205
|
requires_human: requiresHuman,
|
|
121
206
|
};
|
|
122
207
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { bullets, looksLikeUiTask, section } from "./shared.js";
|
|
2
2
|
import { problemSolvingResultContract } from "./problem-solving.js";
|
|
3
|
+
import { acceptanceCriterionRows } from "./completion-contract.js";
|
|
3
4
|
|
|
4
5
|
export function buildCodingInstruction(
|
|
5
6
|
charter,
|
|
@@ -31,7 +32,24 @@ export function buildCodingInstruction(
|
|
|
31
32
|
`Local evidence root: ${charter.ui_evidence_root || "not required"}`,
|
|
32
33
|
].join("\n"),
|
|
33
34
|
),
|
|
34
|
-
section(
|
|
35
|
+
section(
|
|
36
|
+
"completion_contract",
|
|
37
|
+
[
|
|
38
|
+
`User-visible outcome: ${charter.completion_contract?.user_outcome || charter.thread_state.user_goal}`,
|
|
39
|
+
`Minimum scope:\n${bullets(charter.completion_contract?.minimum_scope || charter.acceptance_criteria)}`,
|
|
40
|
+
`Non-goals:\n${bullets(charter.completion_contract?.non_goals || charter.thread_state.non_goals)}`,
|
|
41
|
+
`Proof required:\n${bullets(charter.completion_contract?.proof_required || [])}`,
|
|
42
|
+
`Deadline: ${charter.completion_contract?.deadline || "not specified"}`,
|
|
43
|
+
].join("\n"),
|
|
44
|
+
),
|
|
45
|
+
section(
|
|
46
|
+
"acceptance_criteria",
|
|
47
|
+
bullets(
|
|
48
|
+
acceptanceCriterionRows(charter).map(
|
|
49
|
+
(criterion) => `${criterion.id}: ${criterion.statement}`,
|
|
50
|
+
),
|
|
51
|
+
),
|
|
52
|
+
),
|
|
35
53
|
section("expected_tests", bullets(charter.expected_tests)),
|
|
36
54
|
section("must_not", bullets(charter.must_not)),
|
|
37
55
|
];
|
|
@@ -91,16 +109,17 @@ export function buildCodingInstruction(
|
|
|
91
109
|
"Treat implementation as hypothesis testing: state assumptions, list plausible causes or approaches, inspect discriminating evidence, then choose the smallest strategy supported by that evidence.",
|
|
92
110
|
attemptNumber > 1
|
|
93
111
|
? "This is a retry. Before editing, generate at least eight distinct MECE hypotheses, select a different primary hypothesis or approach, and do not repeat the previous strategy."
|
|
94
|
-
: "For the first attempt,
|
|
112
|
+
: "For the first attempt, identify the single highest uncertainty and run the smallest discriminating verification before editing.",
|
|
95
113
|
attemptNumber > 1
|
|
96
114
|
? "Explain exactly what changed from the previous attempt and why the new evidence supports this strategy."
|
|
97
|
-
: "Record the
|
|
115
|
+
: "Record the uncertainty, first verification, concrete evidence checked, observed result, selected hypothesis, and strategy in the structured result.",
|
|
98
116
|
].join("\n"),
|
|
99
117
|
),
|
|
100
118
|
section(
|
|
101
119
|
"execution_contract",
|
|
102
120
|
[
|
|
103
121
|
"Read the repository instructions and current git state before editing.",
|
|
122
|
+
"Before editing, resolve the highest uncertainty with the first discriminating verification. If it cannot be resolved, stop and report a fallback instead of claiming completion.",
|
|
104
123
|
"Preserve user changes and keep the diff scoped to the requirement.",
|
|
105
124
|
"Run relevant tests, inspect the final diff, commit, push, and create or update a PR.",
|
|
106
125
|
`The PR must target ${charter.base_branch}.`,
|
|
@@ -111,8 +130,11 @@ export function buildCodingInstruction(
|
|
|
111
130
|
"result_contract",
|
|
112
131
|
[
|
|
113
132
|
"Finish with exactly one line beginning SIMY_RESULT_JSON: followed by one JSON object.",
|
|
114
|
-
"Required keys: outcome_kind, summary, branch_name, commit_sha, commit_message_headline, pr_url, pr_number, pr_title, pr_base_branch, tests_run, tests_passed, ui_evidence_path, unrelated_changes_detected, secret_scan_passed, changed_files, residual_risks, problem_solving.",
|
|
133
|
+
"Required keys: outcome_kind, summary, branch_name, commit_sha, commit_message_headline, pr_url, pr_number, pr_title, pr_base_branch, tests_run, tests_passed, ui_evidence_path, unrelated_changes_detected, secret_scan_passed, changed_files, residual_risks, problem_solving, uncertainty_verification, criterion_evidence, visual_review.",
|
|
115
134
|
problemSolvingResultContract(),
|
|
135
|
+
"uncertainty_verification must contain top_uncertainty, first_verification, evidence_checked, observed_result, resolved, and fallback. Only set resolved=true when concrete observed evidence resolves it.",
|
|
136
|
+
"criterion_evidence must contain one entry for every AC-N with criterion_id, passed, verification_method, evidence_refs, and observed_result. Only set passed=true when the evidence directly proves that criterion.",
|
|
137
|
+
"For UI or browser-extension work, visual_review must contain inspected=true, the local screenshot or video paths in artifacts, and concrete acceptance_observations from viewing the actual content. Capturing a file without viewing it does not count.",
|
|
116
138
|
"Use null for unavailable scalar values and [] for unavailable arrays. Do not claim evidence that was not observed.",
|
|
117
139
|
].join("\n"),
|
|
118
140
|
),
|
package/src/orchestrator/loop.js
CHANGED
|
@@ -202,7 +202,11 @@ export async function runCodingLoop({
|
|
|
202
202
|
}),
|
|
203
203
|
);
|
|
204
204
|
if (shouldStop()) return stopCodingLoop(snapshot, onUpdate);
|
|
205
|
-
attempt.independent_audit = buildIndependentAudit(
|
|
205
|
+
attempt.independent_audit = buildIndependentAudit(
|
|
206
|
+
auditExecution,
|
|
207
|
+
snapshot.charter,
|
|
208
|
+
attempt,
|
|
209
|
+
);
|
|
206
210
|
attempt.token_usage = mergeTokenUsage(attempt.token_usage, auditExecution.tokenUsage);
|
|
207
211
|
attempt.budget = refreshBudgetState(snapshot);
|
|
208
212
|
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { cleanString, stringArray } from "./shared.js";
|
|
2
2
|
import { normalizeProblemSolving } from "./problem-solving.js";
|
|
3
|
+
import {
|
|
4
|
+
normalizeCriterionEvidence,
|
|
5
|
+
normalizeUncertaintyVerification,
|
|
6
|
+
normalizeVisualReview,
|
|
7
|
+
} from "./completion-contract.js";
|
|
3
8
|
|
|
4
9
|
export function buildAttempt({ attemptNumber, charter, instruction, promptInterventions, execution }) {
|
|
5
10
|
const result = execution.result && typeof execution.result === "object" ? execution.result : {};
|
|
@@ -28,6 +33,11 @@ export function buildAttempt({ attemptNumber, charter, instruction, promptInterv
|
|
|
28
33
|
ui_evidence_path: cleanString(result.ui_evidence_path) || null,
|
|
29
34
|
summary,
|
|
30
35
|
problem_solving: normalizeProblemSolving(result.problem_solving),
|
|
36
|
+
uncertainty_verification: normalizeUncertaintyVerification(
|
|
37
|
+
result.uncertainty_verification,
|
|
38
|
+
),
|
|
39
|
+
criterion_evidence: normalizeCriterionEvidence(result.criterion_evidence),
|
|
40
|
+
visual_review: normalizeVisualReview(result.visual_review),
|
|
31
41
|
prompt_interventions: promptInterventions,
|
|
32
42
|
work_log_signals: [],
|
|
33
43
|
work_log_interventions: [],
|
package/src/runner.js
CHANGED
|
@@ -158,6 +158,7 @@ export function restoreRun({ snapshot, session, apiOrigin, localPath = null }) {
|
|
|
158
158
|
risk: charter.risk,
|
|
159
159
|
design_review: charter.design_review,
|
|
160
160
|
evidence_policy: charter.evidence_policy,
|
|
161
|
+
completion_contract: charter.completion_contract,
|
|
161
162
|
prompt_policy_report: charter.prompt_policy_report,
|
|
162
163
|
original_request: charter.thread_state?.original_request,
|
|
163
164
|
user_goal: charter.thread_state?.user_goal,
|