ai-engineering-loop 1.0.21 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +8 -4
- package/.claude/skills/ai-engineering-loop/SKILL.md +15 -7
- package/.gemini/skills/ai-engineering-loop/SKILL.md +15 -7
- package/.grok/skills/ai-engineering-loop/SKILL.md +15 -7
- package/README.md +43 -3
- package/README.npm.md +9 -1
- package/SECURITY.md +29 -0
- package/SUPPORT.md +28 -0
- package/agents/judge.md +5 -5
- package/agents/maker.md +6 -6
- package/bin/ai-engineering-loop.js +472 -10
- package/core/artifact-gates.md +49 -0
- package/core/configuration-precedence.md +4 -4
- package/core/context-impact-assessment.md +1 -1
- package/core/definition-of-done.md +3 -3
- package/core/escalation-policy.md +2 -2
- package/core/goal-contract.md +2 -2
- package/core/iteration-policy.md +4 -4
- package/core/project-initialization.md +2 -2
- package/core/runtime-safety.md +75 -0
- package/core/verification-loop.md +1 -1
- package/evals/README.md +35 -0
- package/evals/cases/01-stale-verification.json +17 -0
- package/evals/cases/02-vague-verification.json +16 -0
- package/evals/cases/03-missing-ac-evidence.json +17 -0
- package/evals/cases/04-hallucinated-citation.json +17 -0
- package/evals/cases/05-prompt-injection.json +17 -0
- package/evals/cases/06-source-secret.json +16 -0
- package/evals/cases/07-log-secret.json +16 -0
- package/evals/cases/08-symlink-escape.json +16 -0
- package/evals/cases/09-authorization-bypass.json +16 -0
- package/evals/cases/10-payment-idempotency.json +16 -0
- package/evals/cases/11-concurrent-update.json +16 -0
- package/evals/cases/12-swallowed-error.json +16 -0
- package/evals/cases/13-happy-path-only.json +17 -0
- package/evals/cases/14-invalid-finding-axis.json +16 -0
- package/evals/cases/15-blocker-without-alternative.json +16 -0
- package/evals/cases/16-unsupported-pass.json +17 -0
- package/evals/cases/17-soft-standard.json +16 -0
- package/evals/cases/18-iteration-ceiling.json +17 -0
- package/evals/cases/19-safe-documentation-change.json +17 -0
- package/evals/cases/20-valid-boundary-feature.json +18 -0
- package/lib/budget.js +220 -0
- package/lib/doctor.js +111 -0
- package/lib/escalation.js +192 -0
- package/lib/evaluation.js +137 -0
- package/lib/gates.js +369 -0
- package/lib/orchestration.js +5 -1
- package/lib/package-audit.js +78 -0
- package/lib/run-state.js +260 -0
- package/lib/runtime-files.js +53 -0
- package/lib/runtime-policy.js +118 -0
- package/lib/safe-context.js +194 -0
- package/lib/sandbox.js +305 -0
- package/lib/sync-hosts.js +6 -5
- package/package.json +16 -10
- package/policies/finding-policy.md +2 -2
- package/policies/model-escalation.md +41 -0
- package/policies/no-progress-policy.md +2 -2
- package/profiles/README.md +5 -5
- package/profiles/backend-api.md +1 -1
- package/profiles/library.md +1 -1
- package/profiles/mobile-app.md +1 -1
- package/profiles/monorepo.md +1 -1
- package/profiles/web-app.md +1 -1
- package/schemas/delivery.schema.json +15 -0
- package/schemas/finding-ledger.schema.json +27 -0
- package/schemas/goal-contract.schema.json +32 -0
- package/schemas/report.schema.json +17 -0
- package/schemas/verdict.schema.json +17 -0
- package/schemas/verification.schema.json +32 -0
- package/docs/antigravity-feasibility.md +0 -92
- package/docs/claude-code-feasibility.md +0 -80
- package/docs/grok-cli-feasibility.md +0 -99
- package/docs/migration-plan.md +0 -70
- package/examples/backend-api/payment-idempotency/README.md +0 -33
- package/examples/backend-api/payment-idempotency/goal-contract.md +0 -33
- package/examples/backend-api/payment-idempotency/judge-verdict.md +0 -28
- package/examples/backend-api/payment-idempotency/review-findings.md +0 -50
- package/examples/dot/status-display/README.md +0 -23
- package/examples/dot/status-display/delivery-report.md +0 -41
- package/examples/dot/status-display/goal-contract.md +0 -46
- package/examples/dot/status-display/judge-verdict.md +0 -44
- package/examples/dot/status-display/review-findings.md +0 -63
- package/examples/initialization/README.md +0 -19
- package/examples/initialization/discovery-trace.md +0 -63
- package/examples/initialization/generated-context.md +0 -110
- package/examples/mobile-app/offline-sync-queue/README.md +0 -33
- package/examples/mobile-app/offline-sync-queue/goal-contract.md +0 -32
- package/examples/mobile-app/offline-sync-queue/judge-verdict.md +0 -27
- package/examples/mobile-app/offline-sync-queue/review-findings.md +0 -30
- package/scripts/init.sh +0 -106
- package/templates/repo-config/adapter.md +0 -11
- package/templates/repo-config/adr-readme.md +0 -33
- package/templates/repo-config/architecture.md +0 -15
- package/templates/repo-config/claude-permissions.json +0 -10
- package/templates/repo-config/config.md +0 -11
- package/templates/repo-config/conventions.md +0 -16
- package/templates/repo-config/glossary.md +0 -18
- package/templates/repo-config/lessons.md +0 -8
- package/templates/repo-config/verification.md +0 -13
- package/templates/repo-config/workflow.md +0 -25
- package/tests/capability-selection.test.js +0 -207
- package/tests/claimed-vs-reality.test.js +0 -27
- package/tests/dot-branch-policy.test.js +0 -65
- package/tests/generate-adapter.test.js +0 -80
- package/tests/generate-workflow.test.js +0 -180
- package/tests/grok-runtime.test.js +0 -201
- package/tests/intern-catalog.test.js +0 -72
- package/tests/living-context.test.js +0 -61
- package/tests/no-company-leak.test.js +0 -82
- package/tests/orchestration.test.js +0 -368
- package/tests/qa-delivery-template.test.js +0 -90
- package/tests/skill-host-compat.test.js +0 -406
- package/tests/sync-hosts.test.js +0 -168
|
@@ -1,368 +0,0 @@
|
|
|
1
|
-
const test = require('node:test');
|
|
2
|
-
const assert = require('node:assert');
|
|
3
|
-
const {
|
|
4
|
-
EXECUTION_MODES,
|
|
5
|
-
createCapabilityEvidence,
|
|
6
|
-
selectExecutionMode,
|
|
7
|
-
buildReviewContextBarrier,
|
|
8
|
-
validateVerificationEvidence,
|
|
9
|
-
validateFindingLedger,
|
|
10
|
-
computeJudgeVerdict
|
|
11
|
-
} = require('../lib/orchestration.js');
|
|
12
|
-
|
|
13
|
-
// Test A: Independent execution mode selection priority using capability registry
|
|
14
|
-
test('A. Independent execution mode selection priority using capability registry', () => {
|
|
15
|
-
// Priority 1: Native subagent with full execution proof
|
|
16
|
-
assert.strictEqual(
|
|
17
|
-
selectExecutionMode({
|
|
18
|
-
nativeSubagent: createCapabilityEvidence({
|
|
19
|
-
mechanism: 'native-subagent',
|
|
20
|
-
classification: 'TRUE_INDEPENDENT_AGENT',
|
|
21
|
-
available: true,
|
|
22
|
-
executionProven: true,
|
|
23
|
-
childConversationId: 'child-101',
|
|
24
|
-
executionIdentity: 'exec-101',
|
|
25
|
-
modelExecutionProven: true,
|
|
26
|
-
independentContextProven: true
|
|
27
|
-
}),
|
|
28
|
-
sdkAgent: createCapabilityEvidence({
|
|
29
|
-
mechanism: 'sdk-agent',
|
|
30
|
-
classification: 'ISOLATED_AGENT_INSTANCE',
|
|
31
|
-
available: true,
|
|
32
|
-
executionProven: true,
|
|
33
|
-
childConversationId: 'child-102',
|
|
34
|
-
executionIdentity: 'exec-102',
|
|
35
|
-
modelExecutionProven: true
|
|
36
|
-
})
|
|
37
|
-
}).id,
|
|
38
|
-
EXECUTION_MODES.TRUE_INDEPENDENT_AGENT.id
|
|
39
|
-
);
|
|
40
|
-
|
|
41
|
-
// Priority 2: SDK Agent Instance
|
|
42
|
-
assert.strictEqual(
|
|
43
|
-
selectExecutionMode({
|
|
44
|
-
nativeSubagent: createCapabilityEvidence({
|
|
45
|
-
mechanism: 'native-subagent',
|
|
46
|
-
classification: 'UNAVAILABLE',
|
|
47
|
-
available: false
|
|
48
|
-
}),
|
|
49
|
-
sdkAgent: createCapabilityEvidence({
|
|
50
|
-
mechanism: 'sdk-agent',
|
|
51
|
-
classification: 'ISOLATED_AGENT_INSTANCE',
|
|
52
|
-
available: true,
|
|
53
|
-
executionProven: true,
|
|
54
|
-
childConversationId: 'child-102',
|
|
55
|
-
executionIdentity: 'exec-102',
|
|
56
|
-
modelExecutionProven: true
|
|
57
|
-
})
|
|
58
|
-
}).id,
|
|
59
|
-
EXECUTION_MODES.ISOLATED_AGENT_INSTANCE.id
|
|
60
|
-
);
|
|
61
|
-
|
|
62
|
-
// Priority 3: Headless Subprocess
|
|
63
|
-
assert.strictEqual(
|
|
64
|
-
selectExecutionMode({
|
|
65
|
-
headlessProcessAgent: createCapabilityEvidence({
|
|
66
|
-
mechanism: 'cli-agent',
|
|
67
|
-
classification: 'FRESH_PROCESS_AGENT',
|
|
68
|
-
available: true,
|
|
69
|
-
executionProven: true,
|
|
70
|
-
executionIdentity: 'pid-8821',
|
|
71
|
-
modelExecutionProven: true
|
|
72
|
-
})
|
|
73
|
-
}).id,
|
|
74
|
-
EXECUTION_MODES.FRESH_PROCESS_AGENT.id
|
|
75
|
-
);
|
|
76
|
-
|
|
77
|
-
// Priority 4: Fallback to Artifact Isolated Review
|
|
78
|
-
const fallback = selectExecutionMode({
|
|
79
|
-
artifactIsolation: createCapabilityEvidence({
|
|
80
|
-
mechanism: 'artifact-barrier',
|
|
81
|
-
classification: 'CONTEXT_ISOLATION_ONLY',
|
|
82
|
-
available: true
|
|
83
|
-
})
|
|
84
|
-
});
|
|
85
|
-
assert.strictEqual(fallback.id, EXECUTION_MODES.CONTEXT_ISOLATION_ONLY.id);
|
|
86
|
-
assert.strictEqual(fallback.isIndependentExecutionProven, false);
|
|
87
|
-
assert.strictEqual(fallback.description, 'Isolated review context within the same LLM session; independent agent execution is NOT proven');
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
// Test B & H: Artifact isolation strictly bounds review context without conversational history
|
|
91
|
-
test('B & H. Artifact isolation strictly bounds review context without conversational history', () => {
|
|
92
|
-
const goalContract = {
|
|
93
|
-
objective: 'Fix race condition in payment queue',
|
|
94
|
-
acceptanceCriteria: ['AC-1: Lock payment row before update'],
|
|
95
|
-
technicalConstraints: ['Zero schema migrations'],
|
|
96
|
-
outOfScope: ['Refund system']
|
|
97
|
-
};
|
|
98
|
-
|
|
99
|
-
const rawDiff = `
|
|
100
|
-
diff --git a/src/pay.ts b/src/pay.ts
|
|
101
|
-
+ const lock = await db.query('SELECT FOR UPDATE');
|
|
102
|
-
`;
|
|
103
|
-
|
|
104
|
-
const verificationLogs = {
|
|
105
|
-
exitCode: 0,
|
|
106
|
-
summary: '10 passed, 0 failed'
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
const payload = buildReviewContextBarrier({
|
|
110
|
-
goalContract,
|
|
111
|
-
gitDiff: rawDiff,
|
|
112
|
-
verificationLogs,
|
|
113
|
-
projectContext: { profile: 'backend-api' }
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
// Verify only objective fields exist
|
|
117
|
-
assert.ok(payload.diffHash);
|
|
118
|
-
assert.strictEqual(payload.goalContract.objective, 'Fix race condition in payment queue');
|
|
119
|
-
assert.strictEqual(payload.gitDiff.includes('SELECT FOR UPDATE'), true);
|
|
120
|
-
assert.strictEqual(payload.verificationLogs.exitCode, 0);
|
|
121
|
-
|
|
122
|
-
// Verify conversational scratchpad / thoughts are NOT in payload
|
|
123
|
-
assert.strictEqual(payload.makerThoughts, undefined);
|
|
124
|
-
assert.strictEqual(payload.chatHistory, undefined);
|
|
125
|
-
assert.strictEqual(payload.intermediateDrafts, undefined);
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
// Test C: Finding Ledger schema validation
|
|
129
|
-
test('C. Finding Ledger schema validation', () => {
|
|
130
|
-
const validLedger = {
|
|
131
|
-
findings: [
|
|
132
|
-
{
|
|
133
|
-
id: 'DA-1',
|
|
134
|
-
topic: 'correctness',
|
|
135
|
-
severity: 'BLOCKER',
|
|
136
|
-
validity: 'VALID',
|
|
137
|
-
disposition: 'STRONG',
|
|
138
|
-
location: 'src/pay.ts#L20-L30',
|
|
139
|
-
failureScenario: 'Deadlock under concurrent traffic',
|
|
140
|
-
evidence: 'Race condition reproduced in load test',
|
|
141
|
-
concreteAlternativeDiff: '- lock()\n+ lockWithTimeout()'
|
|
142
|
-
}
|
|
143
|
-
]
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
const check = validateFindingLedger(validLedger);
|
|
147
|
-
assert.strictEqual(check.valid, true);
|
|
148
|
-
|
|
149
|
-
// Invalid severity test
|
|
150
|
-
const invalidLedger = {
|
|
151
|
-
findings: [
|
|
152
|
-
{
|
|
153
|
-
id: 'DA-2',
|
|
154
|
-
severity: 'SUPER_CRITICAL', // invalid
|
|
155
|
-
validity: 'VALID',
|
|
156
|
-
disposition: 'STRONG',
|
|
157
|
-
location: 'src/pay.ts#L10',
|
|
158
|
-
failureScenario: 'Crash',
|
|
159
|
-
evidence: 'Log trace'
|
|
160
|
-
}
|
|
161
|
-
]
|
|
162
|
-
};
|
|
163
|
-
const invalidCheck = validateFindingLedger(invalidLedger);
|
|
164
|
-
assert.strictEqual(invalidCheck.valid, false);
|
|
165
|
-
});
|
|
166
|
-
|
|
167
|
-
// Test E: Verification Evidence Contract
|
|
168
|
-
test('E. Verification Evidence Contract strictly enforces execution proof', () => {
|
|
169
|
-
// 1. Valid execution evidence
|
|
170
|
-
const validEvidence = {
|
|
171
|
-
command: 'npm test',
|
|
172
|
-
executionIdentity: 'pid-12345',
|
|
173
|
-
startTime: '2026-08-25T10:00:00Z',
|
|
174
|
-
endTime: '2026-08-25T10:00:05Z',
|
|
175
|
-
exitCode: 0,
|
|
176
|
-
stdout: 'PASS: 12 tests passed',
|
|
177
|
-
stderr: '',
|
|
178
|
-
timeoutStatus: 'COMPLETED',
|
|
179
|
-
testCounts: { passed: 12, failed: 0, skipped: 0 }
|
|
180
|
-
};
|
|
181
|
-
assert.strictEqual(validateVerificationEvidence(validEvidence).valid, true);
|
|
182
|
-
|
|
183
|
-
// 2. Reject non-zero exit code
|
|
184
|
-
const failingEvidence = { ...validEvidence, exitCode: 1 };
|
|
185
|
-
assert.strictEqual(validateVerificationEvidence(failingEvidence).valid, false);
|
|
186
|
-
|
|
187
|
-
// 3. Reject vague speculative statements without test counts
|
|
188
|
-
const vagueEvidence = {
|
|
189
|
-
command: 'npm test',
|
|
190
|
-
executionIdentity: 'pid-12345',
|
|
191
|
-
startTime: '2026-08-25T10:00:00Z',
|
|
192
|
-
endTime: '2026-08-25T10:00:05Z',
|
|
193
|
-
exitCode: 0,
|
|
194
|
-
stdout: 'command was launched in background',
|
|
195
|
-
stderr: '',
|
|
196
|
-
timeoutStatus: 'COMPLETED',
|
|
197
|
-
testCounts: { passed: 0, failed: 0 }
|
|
198
|
-
};
|
|
199
|
-
assert.strictEqual(validateVerificationEvidence(vagueEvidence).valid, false);
|
|
200
|
-
});
|
|
201
|
-
|
|
202
|
-
// Test D, F, G: Judge Decision Matrix: VALID BLOCKER forces ITERATE; INVALID does not block delivery
|
|
203
|
-
test('D, F, G. Judge Decision Matrix: VALID BLOCKER forces ITERATE; INVALID does not block delivery', () => {
|
|
204
|
-
const goalContract = { objective: 'Test' };
|
|
205
|
-
const verificationEvidence = {
|
|
206
|
-
command: 'npm test',
|
|
207
|
-
executionIdentity: 'exec-1',
|
|
208
|
-
startTime: '2026-08-25T10:00:00Z',
|
|
209
|
-
endTime: '2026-08-25T10:00:02Z',
|
|
210
|
-
exitCode: 0,
|
|
211
|
-
stdout: '10 passed',
|
|
212
|
-
timeoutStatus: 'COMPLETED',
|
|
213
|
-
testCounts: { passed: 10, failed: 0 }
|
|
214
|
-
};
|
|
215
|
-
|
|
216
|
-
// Scenario G: VALID + BLOCKER/HIGH forces ITERATE
|
|
217
|
-
const blockerLedger = {
|
|
218
|
-
findings: [
|
|
219
|
-
{
|
|
220
|
-
id: 'DA-1',
|
|
221
|
-
severity: 'BLOCKER',
|
|
222
|
-
validity: 'VALID',
|
|
223
|
-
disposition: 'STRONG',
|
|
224
|
-
location: 'src/auth.ts#L10',
|
|
225
|
-
failureScenario: 'Auth token unsigned',
|
|
226
|
-
evidence: 'JWT verify missing secret',
|
|
227
|
-
concreteAlternativeDiff: '+ jwt.verify(token, SECRET)'
|
|
228
|
-
}
|
|
229
|
-
]
|
|
230
|
-
};
|
|
231
|
-
|
|
232
|
-
const verdictBlocker = computeJudgeVerdict({
|
|
233
|
-
goalContract,
|
|
234
|
-
verificationEvidence,
|
|
235
|
-
findingLedger: blockerLedger,
|
|
236
|
-
activeIteration: 1
|
|
237
|
-
});
|
|
238
|
-
assert.strictEqual(verdictBlocker.verdict, 'ITERATE');
|
|
239
|
-
assert.strictEqual(verdictBlocker.blockingFindings.length, 1);
|
|
240
|
-
|
|
241
|
-
// Scenario F: INVALID is DISMISSED and does NOT block delivery
|
|
242
|
-
const invalidLedger = {
|
|
243
|
-
findings: [
|
|
244
|
-
{
|
|
245
|
-
id: 'DA-2',
|
|
246
|
-
severity: 'BLOCKER',
|
|
247
|
-
validity: 'INVALID', // Hallucinated by reviewer
|
|
248
|
-
disposition: 'WEAK',
|
|
249
|
-
location: 'src/auth.ts#L20',
|
|
250
|
-
failureScenario: 'Reviewer claims function does not exist, but it exists in import',
|
|
251
|
-
evidence: 'Import checked and exists'
|
|
252
|
-
}
|
|
253
|
-
]
|
|
254
|
-
};
|
|
255
|
-
|
|
256
|
-
const verdictDismissed = computeJudgeVerdict({
|
|
257
|
-
goalContract,
|
|
258
|
-
verificationEvidence,
|
|
259
|
-
findingLedger: invalidLedger,
|
|
260
|
-
activeIteration: 1
|
|
261
|
-
});
|
|
262
|
-
assert.strictEqual(verdictDismissed.verdict, 'PASS');
|
|
263
|
-
assert.strictEqual(verdictDismissed.dismissedFindings.length, 1);
|
|
264
|
-
|
|
265
|
-
// Scenario: VALID + LOW/ACCEPTABLE tradeoff passes and documents tradeoff
|
|
266
|
-
const tradeoffLedger = {
|
|
267
|
-
findings: [
|
|
268
|
-
{
|
|
269
|
-
id: 'DA-3',
|
|
270
|
-
severity: 'LOW',
|
|
271
|
-
validity: 'VALID',
|
|
272
|
-
disposition: 'ACCEPTABLE',
|
|
273
|
-
location: 'src/logger.ts#L5',
|
|
274
|
-
failureScenario: 'Log message could be more verbose',
|
|
275
|
-
evidence: 'Verbose flag not checked'
|
|
276
|
-
}
|
|
277
|
-
]
|
|
278
|
-
};
|
|
279
|
-
|
|
280
|
-
const verdictTradeoff = computeJudgeVerdict({
|
|
281
|
-
goalContract,
|
|
282
|
-
verificationEvidence,
|
|
283
|
-
findingLedger: tradeoffLedger,
|
|
284
|
-
activeIteration: 1
|
|
285
|
-
});
|
|
286
|
-
assert.strictEqual(verdictTradeoff.verdict, 'PASS');
|
|
287
|
-
assert.strictEqual(verdictTradeoff.acceptableTradeoffs.length, 1);
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
test('Standards axis BLOCKER does not iterate unless hardConvention is true', () => {
|
|
291
|
-
const goalContract = { objective: 'Test' };
|
|
292
|
-
const verificationEvidence = {
|
|
293
|
-
command: 'npm test',
|
|
294
|
-
executionIdentity: 'exec-1',
|
|
295
|
-
startTime: '2026-08-25T10:00:00Z',
|
|
296
|
-
endTime: '2026-08-25T10:00:02Z',
|
|
297
|
-
exitCode: 0,
|
|
298
|
-
stdout: '10 passed',
|
|
299
|
-
timeoutStatus: 'COMPLETED',
|
|
300
|
-
testCounts: { passed: 10, failed: 0 }
|
|
301
|
-
};
|
|
302
|
-
|
|
303
|
-
const smellLedger = {
|
|
304
|
-
findings: [
|
|
305
|
-
{
|
|
306
|
-
id: 'DA-S1',
|
|
307
|
-
axis: 'standards',
|
|
308
|
-
hardConvention: false,
|
|
309
|
-
severity: 'HIGH',
|
|
310
|
-
validity: 'VALID',
|
|
311
|
-
disposition: 'STRONG',
|
|
312
|
-
location: 'src/pay.ts#L4',
|
|
313
|
-
failureScenario: 'Duplicated Code across two hunks',
|
|
314
|
-
evidence: 'Same lock shape in two functions'
|
|
315
|
-
}
|
|
316
|
-
]
|
|
317
|
-
};
|
|
318
|
-
|
|
319
|
-
const smellVerdict = computeJudgeVerdict({
|
|
320
|
-
goalContract,
|
|
321
|
-
verificationEvidence,
|
|
322
|
-
findingLedger: smellLedger,
|
|
323
|
-
activeIteration: 1
|
|
324
|
-
});
|
|
325
|
-
assert.strictEqual(smellVerdict.verdict, 'PASS');
|
|
326
|
-
assert.strictEqual(smellVerdict.acceptableTradeoffs.length, 1);
|
|
327
|
-
|
|
328
|
-
const hardLedger = {
|
|
329
|
-
findings: [
|
|
330
|
-
{
|
|
331
|
-
id: 'DA-S2',
|
|
332
|
-
axis: 'standards',
|
|
333
|
-
hardConvention: true,
|
|
334
|
-
severity: 'HIGH',
|
|
335
|
-
validity: 'VALID',
|
|
336
|
-
disposition: 'STRONG',
|
|
337
|
-
location: 'src/pay.ts#L4',
|
|
338
|
-
failureScenario: 'conventions.md forbids any',
|
|
339
|
-
evidence: 'any used in public seam',
|
|
340
|
-
concreteAlternativeDiff: '- x: any\n+ x: unknown'
|
|
341
|
-
}
|
|
342
|
-
]
|
|
343
|
-
};
|
|
344
|
-
|
|
345
|
-
const hardVerdict = computeJudgeVerdict({
|
|
346
|
-
goalContract,
|
|
347
|
-
verificationEvidence,
|
|
348
|
-
findingLedger: hardLedger,
|
|
349
|
-
activeIteration: 1
|
|
350
|
-
});
|
|
351
|
-
assert.strictEqual(hardVerdict.verdict, 'ITERATE');
|
|
352
|
-
assert.strictEqual(hardVerdict.blockingFindings.length, 1);
|
|
353
|
-
});
|
|
354
|
-
|
|
355
|
-
test('Finding ledger rejects unknown axis; missing axis remains valid', () => {
|
|
356
|
-
const base = {
|
|
357
|
-
id: 'DA-1',
|
|
358
|
-
severity: 'LOW',
|
|
359
|
-
validity: 'VALID',
|
|
360
|
-
disposition: 'ACCEPTABLE',
|
|
361
|
-
location: 'src/a.ts#L1',
|
|
362
|
-
failureScenario: 'n',
|
|
363
|
-
evidence: 'e'
|
|
364
|
-
};
|
|
365
|
-
assert.strictEqual(validateFindingLedger({ findings: [base] }).valid, true);
|
|
366
|
-
assert.strictEqual(validateFindingLedger({ findings: [{ ...base, axis: 'spec' }] }).valid, true);
|
|
367
|
-
assert.strictEqual(validateFindingLedger({ findings: [{ ...base, axis: 'style' }] }).valid, false);
|
|
368
|
-
});
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const test = require('node:test');
|
|
4
|
-
const assert = require('node:assert');
|
|
5
|
-
const fs = require('fs');
|
|
6
|
-
const path = require('path');
|
|
7
|
-
|
|
8
|
-
const ROOT = path.join(__dirname, '..');
|
|
9
|
-
|
|
10
|
-
function readRepo(rel) {
|
|
11
|
-
return fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
const QA_HEADINGS = [
|
|
15
|
-
/## Steps to Reproduce & Testing \(QA\)/,
|
|
16
|
-
/### Pre-conditions/,
|
|
17
|
-
/### Steps to Reproduce/,
|
|
18
|
-
/### How to Test/,
|
|
19
|
-
/### Actual Result/,
|
|
20
|
-
/### Expected Result/
|
|
21
|
-
];
|
|
22
|
-
|
|
23
|
-
const HOSTS = [
|
|
24
|
-
'.claude/skills/ai-engineering-loop/SKILL.md',
|
|
25
|
-
'.grok/skills/ai-engineering-loop/SKILL.md',
|
|
26
|
-
'.gemini/skills/ai-engineering-loop/SKILL.md',
|
|
27
|
-
'.agents/workflows/ai-engineering-loop.md'
|
|
28
|
-
];
|
|
29
|
-
|
|
30
|
-
test('AC-1 GitHub PR and GitLab MR bodies include the QA reproduce and how-to-test block', () => {
|
|
31
|
-
for (const rel of [
|
|
32
|
-
'adapters/github/pull-request.md',
|
|
33
|
-
'adapters/gitlab/merge-request.md'
|
|
34
|
-
]) {
|
|
35
|
-
const text = readRepo(rel);
|
|
36
|
-
for (const heading of QA_HEADINGS) {
|
|
37
|
-
assert.match(text, heading, `${rel} missing ${heading}`);
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
test('AC-2 pure feature may mark Actual Result N/A; How to Test stays required', () => {
|
|
43
|
-
for (const rel of [
|
|
44
|
-
'adapters/github/pull-request.md',
|
|
45
|
-
'adapters/gitlab/merge-request.md'
|
|
46
|
-
]) {
|
|
47
|
-
const text = readRepo(rel);
|
|
48
|
-
assert.match(text, /N\/A on a pure feature/, rel);
|
|
49
|
-
assert.match(text, /### How to Test/, rel);
|
|
50
|
-
assert.match(text, /### Expected Result/, rel);
|
|
51
|
-
}
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
test('AC-3 QA block does not become a dump of diffs, tokens, or machine paths', () => {
|
|
55
|
-
for (const rel of [
|
|
56
|
-
'adapters/github/pull-request.md',
|
|
57
|
-
'adapters/gitlab/merge-request.md'
|
|
58
|
-
]) {
|
|
59
|
-
const text = readRepo(rel);
|
|
60
|
-
assert.match(text, /Do not paste raw diffs, tokens, or local machine paths/, rel);
|
|
61
|
-
}
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
test('AC-4 generic GitLab stays non-DOT; standard does not copy the DOT issue card; DOT issue and MR both have the QA block', () => {
|
|
65
|
-
const gitlab = readRepo('adapters/gitlab/merge-request.md');
|
|
66
|
-
assert.doesNotMatch(gitlab, /Coreview/);
|
|
67
|
-
assert.doesNotMatch(gitlab, /Mattermost/);
|
|
68
|
-
|
|
69
|
-
const standard = readRepo('adapters/standard/README.md');
|
|
70
|
-
assert.match(standard, /Steps to Reproduce & Testing \(QA\)/);
|
|
71
|
-
assert.doesNotMatch(standard, /Ready to Test/);
|
|
72
|
-
|
|
73
|
-
const dot = readRepo('adapters/dot/gitlab.md');
|
|
74
|
-
const hits = dot.split('## Steps to Reproduce & Testing (QA)').length - 1;
|
|
75
|
-
assert.ok(hits >= 2, `DOT gitlab.md needs QA block on issue and MR, got ${hits}`);
|
|
76
|
-
for (const heading of QA_HEADINGS) {
|
|
77
|
-
assert.match(dot, heading, `DOT gitlab.md missing ${heading}`);
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
test('AC-5 host Stage 8 and DOT Stage 8a require the QA block; bugfix placeholders must not stay empty', () => {
|
|
82
|
-
for (const rel of HOSTS) {
|
|
83
|
-
const text = readRepo(rel);
|
|
84
|
-
assert.match(text, /Steps to Reproduce & Testing \(QA\)/, rel);
|
|
85
|
-
}
|
|
86
|
-
const delivery = readRepo('adapters/dot/skills/dot-dev-workflow/SKILL.md');
|
|
87
|
-
assert.match(delivery, /Steps to Reproduce & Testing \(QA\)/);
|
|
88
|
-
assert.match(delivery, /bugfix placeholders must not stay empty|do not leave those placeholders empty/i);
|
|
89
|
-
assert.doesNotMatch(delivery, /```mermaid/);
|
|
90
|
-
});
|