ai-engineering-loop 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.full.md +251 -0
- package/README.md +23 -30
- package/agents/devil-advocate.md +54 -92
- package/core/judge-policy.md +48 -101
- package/core/orchestration-model.md +87 -0
- package/core/verification-loop.md +47 -83
- package/lib/orchestration.js +409 -0
- package/package.json +2 -2
- package/policies/evidence-policy.md +33 -15
- package/policies/finding-policy.md +39 -71
- package/tests/capability-selection.test.js +207 -0
- package/tests/orchestration.test.js +288 -0
- package/.README.github.bak.md +0 -272
- package/scripts/publish-npm.js +0 -32
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI Engineering Loop — Runtime Capability Registry & Orchestration Core
|
|
3
|
+
*
|
|
4
|
+
* Invariants:
|
|
5
|
+
* 1. AI Engineering Loop must NEVER claim independent agent execution without runtime
|
|
6
|
+
* evidence of an actual separate LLM execution.
|
|
7
|
+
* 2. Distinction between CONFIGURATION_SUPPORTED, INVOCATION_AVAILABLE, and EXECUTION_PROVEN:
|
|
8
|
+
* - CONFIGURATION_SUPPORTED: The platform understands subagent configuration (e.g. AGENT.md, subagent: true).
|
|
9
|
+
* - INVOCATION_AVAILABLE: An invocation tool or authenticated CLI is callable in the active runtime.
|
|
10
|
+
* - EXECUTION_PROVEN: A separate child session produced an actual model response with independent context.
|
|
11
|
+
* 3. Artifact isolation is strictly reported as CONTEXT_ISOLATION_ONLY (Independent LLM execution: NOT PROVEN).
|
|
12
|
+
* 4. browser_subagent is browser automation and must NOT be classified as an LLM subagent.
|
|
13
|
+
* 5. agentapi send-message is an IPC communication capability and must NEVER activate agent execution.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const crypto = require('crypto');
|
|
17
|
+
|
|
18
|
+
// 1. Standard 5 Execution Modes
|
|
19
|
+
const EXECUTION_MODES = {
|
|
20
|
+
TRUE_INDEPENDENT_AGENT: {
|
|
21
|
+
id: 'TRUE_INDEPENDENT_AGENT',
|
|
22
|
+
label: 'True Independent Agent',
|
|
23
|
+
isIndependentExecutionProven: true,
|
|
24
|
+
description: 'Separate child conversation/process exists, actual LLM execution occurs, and child has independent conversational context'
|
|
25
|
+
},
|
|
26
|
+
ISOLATED_AGENT_INSTANCE: {
|
|
27
|
+
id: 'ISOLATED_AGENT_INSTANCE',
|
|
28
|
+
label: 'Isolated Agent Instance',
|
|
29
|
+
isIndependentExecutionProven: true,
|
|
30
|
+
description: 'Separate conversation or agent instance exists with verified independent model execution'
|
|
31
|
+
},
|
|
32
|
+
FRESH_PROCESS_AGENT: {
|
|
33
|
+
id: 'FRESH_PROCESS_AGENT',
|
|
34
|
+
label: 'Fresh Process Agent',
|
|
35
|
+
isIndependentExecutionProven: true,
|
|
36
|
+
description: 'Separate OS process successfully executes an LLM agent with fresh context'
|
|
37
|
+
},
|
|
38
|
+
CONTEXT_ISOLATION_ONLY: {
|
|
39
|
+
id: 'CONTEXT_ISOLATION_ONLY',
|
|
40
|
+
label: 'Context Isolation Only',
|
|
41
|
+
isIndependentExecutionProven: false,
|
|
42
|
+
description: 'Isolated review context within the same LLM session; independent agent execution is NOT proven'
|
|
43
|
+
},
|
|
44
|
+
UNAVAILABLE: {
|
|
45
|
+
id: 'UNAVAILABLE',
|
|
46
|
+
label: 'Unavailable',
|
|
47
|
+
isIndependentExecutionProven: false,
|
|
48
|
+
description: 'No review execution mechanism is available'
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* 2. Standardized Capability Evidence Factory with 3-Stage Lifecycle
|
|
54
|
+
*/
|
|
55
|
+
function createCapabilityEvidence({
|
|
56
|
+
mechanism,
|
|
57
|
+
classification,
|
|
58
|
+
configurationSupported = false,
|
|
59
|
+
invocationAvailable = false,
|
|
60
|
+
executionProven = false,
|
|
61
|
+
available = false,
|
|
62
|
+
commandOrApi = null,
|
|
63
|
+
result = null,
|
|
64
|
+
executionIdentity = null,
|
|
65
|
+
conversationId = null,
|
|
66
|
+
parentConversationId = null,
|
|
67
|
+
childConversationId = null,
|
|
68
|
+
childModelResponse = null,
|
|
69
|
+
modelExecutionProven = false,
|
|
70
|
+
independentContextProven = false,
|
|
71
|
+
historyInherited = null,
|
|
72
|
+
isDocumentationOnly = false,
|
|
73
|
+
isBrowserAutomationOnly = false,
|
|
74
|
+
reason = null,
|
|
75
|
+
timestamp = new Date().toISOString()
|
|
76
|
+
}) {
|
|
77
|
+
if (!mechanism || !classification) {
|
|
78
|
+
throw new Error('Capability evidence must include mechanism and classification');
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const modelExecuted = Boolean(modelExecutionProven || (childModelResponse && executionIdentity));
|
|
82
|
+
const fullyProven = Boolean(executionProven || (modelExecuted && childConversationId && independentContextProven));
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
mechanism,
|
|
86
|
+
classification,
|
|
87
|
+
configurationSupported: Boolean(configurationSupported),
|
|
88
|
+
invocationAvailable: Boolean(invocationAvailable),
|
|
89
|
+
executionProven: fullyProven,
|
|
90
|
+
available: Boolean(available || fullyProven),
|
|
91
|
+
commandOrApi,
|
|
92
|
+
result,
|
|
93
|
+
executionIdentity,
|
|
94
|
+
conversationId,
|
|
95
|
+
parentConversationId,
|
|
96
|
+
childConversationId,
|
|
97
|
+
childModelResponse,
|
|
98
|
+
modelExecutionProven: modelExecuted,
|
|
99
|
+
independentContextProven: Boolean(independentContextProven),
|
|
100
|
+
historyInherited,
|
|
101
|
+
isDocumentationOnly: Boolean(isDocumentationOnly),
|
|
102
|
+
isBrowserAutomationOnly: Boolean(isBrowserAutomationOnly),
|
|
103
|
+
reason,
|
|
104
|
+
timestamp
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* 3. Evaluate whether a capability candidate satisfies execution proof
|
|
110
|
+
*/
|
|
111
|
+
function isCapabilityProvenForMode(evidence, targetModeId) {
|
|
112
|
+
if (!evidence) {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Reject non-agent capabilities (such as IPC send-message)
|
|
117
|
+
if (
|
|
118
|
+
evidence.classification === 'NOT_AN_AGENT_EXECUTION_CAPABILITY' ||
|
|
119
|
+
evidence.classification === 'IPC_MESSAGE_DISPATCH'
|
|
120
|
+
) {
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Reject browser automation tools (browser_subagent is NOT an LLM subagent)
|
|
125
|
+
if (evidence.isBrowserAutomationOnly || evidence.classification === 'BROWSER_AUTOMATION_TOOL') {
|
|
126
|
+
return false;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Reject documentation-only or configuration-only claims
|
|
130
|
+
if (evidence.isDocumentationOnly || (evidence.configurationSupported && !evidence.executionProven)) {
|
|
131
|
+
if (targetModeId !== 'CONTEXT_ISOLATION_ONLY') {
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Modes requiring true independent LLM execution
|
|
137
|
+
if (
|
|
138
|
+
targetModeId === 'TRUE_INDEPENDENT_AGENT' ||
|
|
139
|
+
targetModeId === 'ISOLATED_AGENT_INSTANCE' ||
|
|
140
|
+
targetModeId === 'FRESH_PROCESS_AGENT'
|
|
141
|
+
) {
|
|
142
|
+
// Must satisfy all 4 pillars of execution proof:
|
|
143
|
+
const hasChildSession = Boolean(evidence.childConversationId || targetModeId === 'FRESH_PROCESS_AGENT');
|
|
144
|
+
const hasModelResponse = Boolean(evidence.modelExecutionProven || evidence.childModelResponse);
|
|
145
|
+
const hasExecIdentity = Boolean(evidence.executionIdentity);
|
|
146
|
+
const hasCleanContext = Boolean(evidence.independentContextProven && evidence.historyInherited !== true);
|
|
147
|
+
|
|
148
|
+
if (!evidence.executionProven && !(hasChildSession && hasModelResponse && hasExecIdentity && hasCleanContext)) {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Context isolation mode requires verified artifact boundary
|
|
154
|
+
if (targetModeId === 'CONTEXT_ISOLATION_ONLY') {
|
|
155
|
+
return evidence.available === true;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return true;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* 4. Deterministic Execution Mode Selection
|
|
163
|
+
* Evaluates registry strictly in priority order:
|
|
164
|
+
* 1. TRUE_INDEPENDENT_AGENT
|
|
165
|
+
* 2. ISOLATED_AGENT_INSTANCE
|
|
166
|
+
* 3. FRESH_PROCESS_AGENT
|
|
167
|
+
* 4. CONTEXT_ISOLATION_ONLY
|
|
168
|
+
* 5. UNAVAILABLE
|
|
169
|
+
*/
|
|
170
|
+
function selectExecutionMode(capabilityRegistry = {}) {
|
|
171
|
+
const {
|
|
172
|
+
nativeSubagent,
|
|
173
|
+
sdkAgent,
|
|
174
|
+
headlessProcessAgent,
|
|
175
|
+
artifactIsolation
|
|
176
|
+
} = capabilityRegistry;
|
|
177
|
+
|
|
178
|
+
// Priority 1: True Native Independent Sub-Agent
|
|
179
|
+
if (isCapabilityProvenForMode(nativeSubagent, 'TRUE_INDEPENDENT_AGENT')) {
|
|
180
|
+
return EXECUTION_MODES.TRUE_INDEPENDENT_AGENT;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Priority 2: Programmatic SDK Agent Instance
|
|
184
|
+
if (isCapabilityProvenForMode(sdkAgent, 'ISOLATED_AGENT_INSTANCE')) {
|
|
185
|
+
return EXECUTION_MODES.ISOLATED_AGENT_INSTANCE;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Priority 3: Headless Subprocess Agent
|
|
189
|
+
if (isCapabilityProvenForMode(headlessProcessAgent, 'FRESH_PROCESS_AGENT')) {
|
|
190
|
+
return EXECUTION_MODES.FRESH_PROCESS_AGENT;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Priority 4: Guaranteed Fallback - Context Isolation Only
|
|
194
|
+
if (isCapabilityProvenForMode(artifactIsolation, 'CONTEXT_ISOLATION_ONLY')) {
|
|
195
|
+
return EXECUTION_MODES.CONTEXT_ISOLATION_ONLY;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Priority 5: Unavailable
|
|
199
|
+
return EXECUTION_MODES.UNAVAILABLE;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* 5. Build Clean-Slate Artifact Isolation Barrier
|
|
204
|
+
*/
|
|
205
|
+
function buildReviewContextBarrier({
|
|
206
|
+
goalContract,
|
|
207
|
+
gitDiff,
|
|
208
|
+
verificationLogs,
|
|
209
|
+
projectContext,
|
|
210
|
+
activeIteration = 1,
|
|
211
|
+
priorFindingSignatures = []
|
|
212
|
+
}) {
|
|
213
|
+
if (!goalContract || !gitDiff) {
|
|
214
|
+
throw new Error('Goal contract and git diff are mandatory for review barrier');
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
return {
|
|
218
|
+
iteration: activeIteration,
|
|
219
|
+
diffHash: crypto.createHash('sha256').update(gitDiff).digest('hex').slice(0, 16),
|
|
220
|
+
goalContract: {
|
|
221
|
+
objective: goalContract.objective,
|
|
222
|
+
acceptanceCriteria: goalContract.acceptanceCriteria || [],
|
|
223
|
+
technicalConstraints: goalContract.technicalConstraints || [],
|
|
224
|
+
outOfScope: goalContract.outOfScope || []
|
|
225
|
+
},
|
|
226
|
+
projectContext: {
|
|
227
|
+
profile: projectContext?.profile || 'standard',
|
|
228
|
+
architecture: projectContext?.architecture || '',
|
|
229
|
+
conventions: projectContext?.conventions || '',
|
|
230
|
+
verificationCommands: projectContext?.verificationCommands || ''
|
|
231
|
+
},
|
|
232
|
+
gitDiff: gitDiff.trim(),
|
|
233
|
+
verificationLogs: {
|
|
234
|
+
exitCode: verificationLogs?.exitCode ?? 0,
|
|
235
|
+
summary: verificationLogs?.summary || 'Deterministic checks passed 100%'
|
|
236
|
+
},
|
|
237
|
+
priorFindingSignatures: [...priorFindingSignatures]
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* 6. Validate Verification Evidence Contract
|
|
243
|
+
*/
|
|
244
|
+
function validateVerificationEvidence(evidence) {
|
|
245
|
+
if (!evidence) return { valid: false, reason: 'Evidence object missing' };
|
|
246
|
+
|
|
247
|
+
const requiredFields = ['command', 'executionIdentity', 'startTime', 'endTime', 'exitCode', 'stdout', 'timeoutStatus'];
|
|
248
|
+
for (const field of requiredFields) {
|
|
249
|
+
if (evidence[field] === undefined || evidence[field] === null || evidence[field] === '') {
|
|
250
|
+
return { valid: false, reason: `Missing required evidence field: ${field}` };
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
if (evidence.exitCode !== 0) {
|
|
255
|
+
return { valid: false, reason: `Verification failed with non-zero exit code: ${evidence.exitCode}` };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if (evidence.timeoutStatus !== 'COMPLETED') {
|
|
259
|
+
return { valid: false, reason: `Verification did not complete cleanly: ${evidence.timeoutStatus}` };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const vaguePatterns = ['command was launched', 'test appears to have passed', 'running in background', 'seems green'];
|
|
263
|
+
const outLower = (evidence.stdout + ' ' + (evidence.summary || '')).toLowerCase();
|
|
264
|
+
for (const pattern of vaguePatterns) {
|
|
265
|
+
if (outLower.includes(pattern) && (!evidence.testCounts || evidence.testCounts.passed === 0)) {
|
|
266
|
+
return { valid: false, reason: `Rejected vague or speculative assertion: "${pattern}"` };
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
return { valid: true };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* 7. Validate Finding Ledger Schema
|
|
275
|
+
*/
|
|
276
|
+
function validateFindingLedger(ledger) {
|
|
277
|
+
if (!ledger || !Array.isArray(ledger.findings)) {
|
|
278
|
+
return { valid: false, reason: 'Finding ledger must contain an array of findings' };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const validSeverities = ['BLOCKER', 'HIGH', 'MEDIUM', 'LOW'];
|
|
282
|
+
const validValidities = ['VALID', 'INVALID'];
|
|
283
|
+
const validDispositions = ['STRONG', 'ACCEPTABLE', 'WEAK'];
|
|
284
|
+
|
|
285
|
+
for (let i = 0; i < ledger.findings.length; i++) {
|
|
286
|
+
const f = ledger.findings[i];
|
|
287
|
+
if (!f.id || !f.location || !f.failureScenario || !f.evidence) {
|
|
288
|
+
return { valid: false, reason: `Finding at index ${i} missing required id, location, failureScenario, or evidence` };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (!validSeverities.includes(f.severity)) {
|
|
292
|
+
return { valid: false, reason: `Invalid severity at index ${i}: ${f.severity}. Must be: ${validSeverities.join(', ')}` };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
if (!validValidities.includes(f.validity)) {
|
|
296
|
+
return { valid: false, reason: `Invalid validity at index ${i}: ${f.validity}. Must be VALID or INVALID` };
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (!validDispositions.includes(f.disposition)) {
|
|
300
|
+
return { valid: false, reason: `Invalid disposition at index ${i}: ${f.disposition}. Must be STRONG, ACCEPTABLE, or WEAK` };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
if (f.validity === 'VALID' && (f.severity === 'BLOCKER' || f.severity === 'HIGH') && !f.concreteAlternativeDiff) {
|
|
304
|
+
return { valid: false, reason: `VALID ${f.severity} finding ${f.id} must provide concreteAlternativeDiff` };
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
return { valid: true };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* 8. Compute Judge Verdict based primarily on Validity + Severity
|
|
313
|
+
*/
|
|
314
|
+
function computeJudgeVerdict({
|
|
315
|
+
goalContract,
|
|
316
|
+
verificationEvidence,
|
|
317
|
+
findingLedger,
|
|
318
|
+
activeIteration = 1,
|
|
319
|
+
maxIterations = 3
|
|
320
|
+
}) {
|
|
321
|
+
const verifCheck = validateVerificationEvidence(verificationEvidence);
|
|
322
|
+
if (!verifCheck.valid) {
|
|
323
|
+
return {
|
|
324
|
+
verdict: 'ITERATE',
|
|
325
|
+
reason: `Deterministic verification failed: ${verifCheck.reason}`,
|
|
326
|
+
action: 'Maker must rerun deterministic test suite and obtain clean exit code 0.'
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const findings = findingLedger?.findings || [];
|
|
331
|
+
const blockingFindings = [];
|
|
332
|
+
const acceptableFindings = [];
|
|
333
|
+
const dismissedFindings = [];
|
|
334
|
+
|
|
335
|
+
for (const f of findings) {
|
|
336
|
+
if (f.validity === 'INVALID') {
|
|
337
|
+
dismissedFindings.push({
|
|
338
|
+
id: f.id,
|
|
339
|
+
reason: `Dismissed invalid finding: ${f.failureScenario} (Evidence disproved)`
|
|
340
|
+
});
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
if (f.severity === 'BLOCKER' || f.severity === 'HIGH') {
|
|
345
|
+
blockingFindings.push(f);
|
|
346
|
+
} else {
|
|
347
|
+
acceptableFindings.push(f);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (blockingFindings.length > 0) {
|
|
352
|
+
if (activeIteration >= maxIterations) {
|
|
353
|
+
return {
|
|
354
|
+
verdict: 'ESCALATE',
|
|
355
|
+
reason: `Max iteration ceiling (${maxIterations}) reached with ${blockingFindings.length} open blocking findings.`,
|
|
356
|
+
blockingFindings,
|
|
357
|
+
action: 'Human escalation triggered with actionable diagnostic ledger.'
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
return {
|
|
362
|
+
verdict: 'ITERATE',
|
|
363
|
+
reason: `${blockingFindings.length} VALID BLOCKER/HIGH findings require code revision.`,
|
|
364
|
+
blockingFindings,
|
|
365
|
+
action: 'Maker must apply concrete alternative diffs and author regression tests.'
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return {
|
|
370
|
+
verdict: 'PASS',
|
|
371
|
+
reason: `All acceptance criteria verified; 0 open blocking findings; ${dismissedFindings.length} invalid findings dismissed; ${acceptableFindings.length} acceptable tradeoffs documented.`,
|
|
372
|
+
acceptableTradeoffs: acceptableFindings,
|
|
373
|
+
dismissedFindings,
|
|
374
|
+
action: 'Proceed to Context Impact Assessment and Delivery Adapter.'
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* 9. Truthful Review Report Generator
|
|
380
|
+
* Enforces mandatory reporting headers and strictly prevents misleading phrasing.
|
|
381
|
+
*/
|
|
382
|
+
function formatExecutionReport({ selectedMode, capabilityRegistry = {} }) {
|
|
383
|
+
const isContextOnly = selectedMode.id === EXECUTION_MODES.CONTEXT_ISOLATION_ONLY.id;
|
|
384
|
+
|
|
385
|
+
const nativeSubagentState = capabilityRegistry.nativeSubagent?.invocationAvailable
|
|
386
|
+
? 'AVAILABLE'
|
|
387
|
+
: 'UNAVAILABLE';
|
|
388
|
+
|
|
389
|
+
const lines = [
|
|
390
|
+
`Execution Mode: ${selectedMode.id}`,
|
|
391
|
+
`Independent LLM Execution: ${selectedMode.isIndependentExecutionProven ? 'PROVEN' : 'NOT PROVEN'}`,
|
|
392
|
+
`Native Subagent Invocation: ${nativeSubagentState}`,
|
|
393
|
+
`Review Method: ${isContextOnly ? 'Clean-Slate Artifact Isolation Barrier' : selectedMode.label}`
|
|
394
|
+
];
|
|
395
|
+
|
|
396
|
+
return lines.join('\n');
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
module.exports = {
|
|
400
|
+
EXECUTION_MODES,
|
|
401
|
+
createCapabilityEvidence,
|
|
402
|
+
isCapabilityProvenForMode,
|
|
403
|
+
selectExecutionMode,
|
|
404
|
+
buildReviewContextBarrier,
|
|
405
|
+
validateVerificationEvidence,
|
|
406
|
+
validateFindingLedger,
|
|
407
|
+
computeJudgeVerdict,
|
|
408
|
+
formatExecutionReport
|
|
409
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-engineering-loop",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.4",
|
|
4
4
|
"description": "A reusable, framework-agnostic AI Engineering Operating System for autonomous coding agents.",
|
|
5
5
|
"main": "bin/ai-engineering-loop.js",
|
|
6
6
|
"bin": {
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
},
|
|
9
9
|
"scripts": {
|
|
10
10
|
"init": "node bin/ai-engineering-loop.js",
|
|
11
|
-
"
|
|
11
|
+
"test": "node --test tests/*.test.js"
|
|
12
12
|
},
|
|
13
13
|
"repository": {
|
|
14
14
|
"type": "git",
|
|
@@ -10,9 +10,23 @@ An agent may not state that code is functional, bug-free, optimized, or ready fo
|
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|
|
13
|
-
## 2.
|
|
13
|
+
## 2. Verification Evidence Contract
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
A verification `PASS` is strictly invalid without concrete execution evidence. The system categorically rejects vague statements such as *"command was launched"* or *"test appears to have passed"*.
|
|
16
|
+
|
|
17
|
+
### Mandatory Execution Evidence Properties:
|
|
18
|
+
1. **`command`**: Exact CLI command executed.
|
|
19
|
+
2. **`executionIdentity`**: Execution ID / PID / timestamp.
|
|
20
|
+
3. **`startTime` & `endTime`**: Documenting execution duration.
|
|
21
|
+
4. **`exitCode`**: Must be `0`.
|
|
22
|
+
5. **`stdout` & `stderr`**: Raw machine logs captured.
|
|
23
|
+
6. **`timeoutStatus`**: Must be `"COMPLETED"`.
|
|
24
|
+
7. **`testCounts`**: Explicit counts of passed, failed, and skipped tests.
|
|
25
|
+
8. **`assertionEvidence`**: Specific assertion proof matching the active Goal Contract's Acceptance Criteria.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## 3. The Evidence Hierarchy
|
|
16
30
|
|
|
17
31
|
```mermaid
|
|
18
32
|
flowchart TD
|
|
@@ -20,7 +34,7 @@ flowchart TD
|
|
|
20
34
|
E2[Level 2: Concrete Codebase Artifacts<br>Actual file contents, git commit history, schema files]
|
|
21
35
|
E3[Level 3: Adversarial Code Diffs<br>Exact before/after reproduction snippets]
|
|
22
36
|
E4[Level 4: Theoretical / Analytical Deduction<br>Reasoning through architectural implications]
|
|
23
|
-
E5[Level 5: Unsupported Assertion<br>'Looks correct', '
|
|
37
|
+
E5[Level 5: Unsupported Assertion<br>'Looks correct', 'Command launched' - REJECTED]
|
|
24
38
|
|
|
25
39
|
E1 --> E2 --> E3 --> E4 -.-> E5
|
|
26
40
|
```
|
|
@@ -49,22 +63,26 @@ flowchart TD
|
|
|
49
63
|
|
|
50
64
|
---
|
|
51
65
|
|
|
52
|
-
##
|
|
66
|
+
## 4. Capability Discovery & Orchestration Evidence Rules
|
|
67
|
+
|
|
68
|
+
To prevent misleading claims of multi-agent execution, capability selection must adhere to strict evidence standards:
|
|
69
|
+
|
|
70
|
+
1. **Documentation is NOT capability proof**: Mentioning `subagent: true` in markdown is not proof of subagent execution.
|
|
71
|
+
2. **Configuration is NOT execution proof**: A YAML config file is not evidence of a running agent.
|
|
72
|
+
3. **IPC existence is NOT execution proof**: The existence of an IPC binary (`agentapi`) or successful message dispatch (`send-message`) is an IPC capability, **NOT** an LLM agent execution capability.
|
|
73
|
+
4. **Conversation creation without model response is NOT an agent**: A conversation ID without an actual captured LLM response is not execution proof.
|
|
74
|
+
5. **Persona simulation is NEVER an agent**: Switching roles within the same session is self-review, not independent adversarial review.
|
|
75
|
+
6. **Artifact isolation is strictly labeled**: When review is conducted within the same session via artifact boundaries, it is **strictly reported as `CONTEXT_ISOLATION_ONLY`** with `Independent LLM execution: NOT PROVEN`.
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## 5. Evidence Requirements for Common Agent Claims
|
|
53
80
|
|
|
54
81
|
| Claim Type | Mandatory Evidence Required | Prohibited Substitute |
|
|
55
82
|
|---|---|---|
|
|
56
83
|
| **"Bug is fixed"** | Reproducing test that previously failed now passes with exit code 0. | "The logic was corrected." |
|
|
57
84
|
| **"No regressions"** | Full test suite execution log showing 0 failures. | "I only touched a single function." |
|
|
58
85
|
| **"Type safe"** | `tsc --noEmit` / compiler run output with 0 errors. | "I added type annotations." |
|
|
59
|
-
| **"Review finding is invalid
|
|
86
|
+
| **"Review finding is invalid"** | File path & line showing the suggested API does not exist or behavior is intentional. | "I disagree with the reviewer." |
|
|
60
87
|
| **"Acceptance criteria AC-X met"** | Test function name & assertion specifically targeting AC-X. | "Implemented according to spec." |
|
|
61
|
-
|
|
62
|
-
---
|
|
63
|
-
|
|
64
|
-
## 4. Evidentiary Audit Trail
|
|
65
|
-
|
|
66
|
-
Every artifact created during the engineering loop (Goal Contract, Maker Log, Review Findings, Judge Verdict) must maintain an unbroken chain of evidence:
|
|
67
|
-
|
|
68
|
-
1. **Exact Commands Run**: Included in verbatim code blocks.
|
|
69
|
-
2. **Standard Output / Error Logs**: Verbatim output snippets without truncation of error counts.
|
|
70
|
-
3. **Traceable File Links**: Every cited file must use clickable format (`file:///path/to/file#L1-L10`).
|
|
88
|
+
| **"Independent subagent executed"** | Verified parent $\rightarrow$ child session ID + separate process + captured child model response. | "I switched to Devil's Advocate persona." |
|
|
@@ -1,102 +1,70 @@
|
|
|
1
1
|
# Finding Policy Specification
|
|
2
2
|
|
|
3
|
-
## 1. Purpose
|
|
3
|
+
## 1. Purpose & Core Philosophy
|
|
4
4
|
|
|
5
|
-
The **Finding Policy** defines the standardized schema, severity definitions,
|
|
5
|
+
The **Finding Policy** defines the standardized schema, severity definitions, validity rules, and lifecycle states for all issues identified by review agents (such as the [Devil's Advocate](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/devil-advocate.md) or external review bots).
|
|
6
6
|
|
|
7
7
|
A standardized finding schema ensures that:
|
|
8
|
-
1. Every criticism is actionable, localized, and backed by evidence.
|
|
8
|
+
1. Every criticism is actionable, localized, and backed by factual code evidence.
|
|
9
9
|
2. Review findings can be parsed, hashed, and tracked across autonomous iterations.
|
|
10
|
-
3. The [Judge Agent](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/judge.md) can evaluate findings
|
|
10
|
+
3. The [Judge Agent](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/judge.md) can evaluate findings deterministically based on **Validity + Severity** without subjective bias.
|
|
11
11
|
|
|
12
12
|
---
|
|
13
13
|
|
|
14
|
-
## 2. Standardized Finding Schema
|
|
14
|
+
## 2. Standardized Dual-Axis Finding Schema
|
|
15
15
|
|
|
16
|
-
Every finding MUST be structured according to the following specification:
|
|
16
|
+
Every finding MUST be structured according to the following dual-axis specification:
|
|
17
17
|
|
|
18
18
|
```yaml
|
|
19
19
|
id: "<CATEGORY_PREFIX>-<3_DIGIT_NUMBER>" # e.g. COR-001, SEC-002, PERF-001
|
|
20
20
|
title: "<Short, descriptive summary of the problem>"
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
topic: "<correctness | error_handling | security | concurrency | performance | maintainability | testing_gaps>"
|
|
22
|
+
|
|
23
|
+
# Axis 1: Factual Validity
|
|
24
|
+
validity: "<VALID | INVALID>" # VALID: Real technical flaw | INVALID: Reviewer hallucination or misunderstanding
|
|
25
|
+
|
|
26
|
+
# Axis 2: Objective Severity
|
|
27
|
+
severity: "<BLOCKER | HIGH | MEDIUM | LOW>"
|
|
28
|
+
|
|
29
|
+
# Reviewer Disposition / Recommendation
|
|
30
|
+
disposition: "<STRONG | ACCEPTABLE | WEAK>"
|
|
31
|
+
|
|
23
32
|
location:
|
|
24
33
|
file: "<Relative file path>"
|
|
25
34
|
startLine: <Integer>
|
|
26
35
|
endLine: <Integer>
|
|
36
|
+
|
|
37
|
+
acceptanceCriteria: "<AC-1..N impacted, or 'GENERAL_REGRESSION'>"
|
|
27
38
|
evidence: "<Exact code snippet, command output, or trace exhibiting the flaw>"
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
39
|
+
failureScenario: "<Concrete step-by-step failure trace describing what breaks at runtime>"
|
|
40
|
+
reproduction: "<Executable test case or sequence demonstrating the bug>"
|
|
41
|
+
concreteAlternativeDiff: |
|
|
42
|
+
```diff
|
|
43
|
+
- old_flawed_code()
|
|
44
|
+
+ new_verified_fix()
|
|
45
|
+
```
|
|
34
46
|
```
|
|
35
47
|
|
|
36
48
|
---
|
|
37
49
|
|
|
38
|
-
## 3. Severity Matrix &
|
|
50
|
+
## 3. Severity Matrix & Decision Impact
|
|
39
51
|
|
|
40
|
-
| Severity Level | Definition | Impact on
|
|
52
|
+
| Severity Level | Definition | Impact on Verdict |
|
|
41
53
|
|---|---|---|
|
|
42
|
-
| **`
|
|
43
|
-
| **`HIGH
|
|
44
|
-
| **`MEDIUM
|
|
45
|
-
| **`LOW
|
|
46
|
-
| **`INFO` (SEV-5)** | Informational observation, architectural note, or praise for a well-designed pattern. | **INFORMATIONAL**: No action required. |
|
|
47
|
-
|
|
48
|
-
---
|
|
49
|
-
|
|
50
|
-
## 4. Review Categories
|
|
51
|
-
|
|
52
|
-
Findings are categorized under one of the 7 standard domains:
|
|
53
|
-
|
|
54
|
-
1. **`Correctness` (`COR`)**: Functional bugs, logic errors, type mismatches, inverted conditionals, timezone bugs, calculation errors.
|
|
55
|
-
2. **`ErrorHandling` (`ERR`)**: Swallowed errors, unhandled rejections, missing fallbacks, crashing async boundaries.
|
|
56
|
-
3. **`Security` (`SEC`)**: Injection, broken auth/access control, secret exposure, sensitive data logging, unsanitized inputs.
|
|
57
|
-
4. **`Concurrency` (`CONC`)**: Race conditions, un-synchronized shared memory, non-atomic database operations, thread safety issues.
|
|
58
|
-
5. **`Performance` (`PERF`)**: N+1 queries, memory leaks, unbounded array loops, unindexed filters, heavy synchronous blocking operations.
|
|
59
|
-
6. **`Maintainability` (`MAINT`)**: High coupling, breaking existing architecture, anti-patterns, dead code, circular dependencies.
|
|
60
|
-
7. **`TestingGaps` (`TEST`)**: Untested boundary conditions, weak/tautological assertions, missing negative test cases.
|
|
54
|
+
| **`BLOCKER`** | System crash, severe data corruption, auth bypass, or direct violation of an explicit Acceptance Criterion. | **BLOCKING**: Forces `ITERATE` verdict. Cannot pass DoD. |
|
|
55
|
+
| **`HIGH`** | Core business logic defect, unhandled null exception on primary user path, or critical performance regression. | **BLOCKING**: Forces `ITERATE` verdict. Must be fixed with regression test. |
|
|
56
|
+
| **`MEDIUM`** | Edge-case logic failure, missing error telemetry, or suboptimal query with limited dataset. | **TRADE-OFF**: Merged if ACs are met; documented as acceptable tradeoff in MR. |
|
|
57
|
+
| **`LOW`** | Minor maintainability issue, non-critical testing gap, minor code duplication, or naming ambiguity. | **TRADE-OFF**: Merged; documented in MR notes. |
|
|
61
58
|
|
|
62
59
|
---
|
|
63
60
|
|
|
64
|
-
##
|
|
65
|
-
|
|
66
|
-
Review findings are hypotheses, not absolute truths. Every finding exists in one of the following states:
|
|
61
|
+
## 4. Validity Rules & Falsification Principle
|
|
67
62
|
|
|
68
|
-
|
|
69
|
-
- **`TRIAGED_VALID`**: Author and Judge confirm that the defect is real and violates technical or contract requirements. Must be fixed by Maker.
|
|
70
|
-
- **`TRIAGED_INVALID` (False Positive / "Halu")**: Author demonstrates with evidence that the reviewer's concern is factually incorrect, based on an API that does not exist, or flags intentional/designed behavior.
|
|
71
|
-
- **`TRIAGED_UNCERTAIN`**: Ambiguous requirement where the codebase documentation is silent or contradictory. Triggers escalation.
|
|
72
|
-
- **`RESOLVED`**: Maker has applied code fix and passing test suite.
|
|
73
|
-
- **`VERIFIED`**: Devil's Advocate and Judge re-evaluated and confirmed the issue is fully solved.
|
|
74
|
-
|
|
75
|
-
---
|
|
63
|
+
Review findings are hypotheses, not absolute truths.
|
|
76
64
|
|
|
77
|
-
|
|
65
|
+
- **`VALID`**: The defect exists in the code and produces an unhandled failure or contract breach under realistic conditions.
|
|
66
|
+
- **`INVALID` (Dismissed)**: The reviewer made an assumption disproved by the codebase, referenced a non-existent API, or flagged intentional behavior.
|
|
78
67
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
severity: "HIGH"
|
|
83
|
-
category: "Correctness"
|
|
84
|
-
location:
|
|
85
|
-
file: "src/server/attendance/utils/calculate-duration.ts"
|
|
86
|
-
startLine: 28
|
|
87
|
-
endLine: 34
|
|
88
|
-
evidence: |
|
|
89
|
-
const checkInHour = new Date(record.clockIn).getHours();
|
|
90
|
-
problem: |
|
|
91
|
-
Using Date.prototype.getHours() extracts the local server timezone hour instead of converting to the employee's designated timezone (e.g. Asia/Jakarta).
|
|
92
|
-
impact: |
|
|
93
|
-
Employees clocking in between 00:00 and 07:00 UTC will be marked as late or absent depending on the server hosting region.
|
|
94
|
-
recommendation: |
|
|
95
|
-
Use timezone-aware date parsing with dayjs/date-fns-tz:
|
|
96
|
-
```diff
|
|
97
|
-
- const checkInHour = new Date(record.clockIn).getHours();
|
|
98
|
-
+ const checkInHour = dayjs(record.clockIn).tz(userTimezone).hour();
|
|
99
|
-
```
|
|
100
|
-
confidence: "HIGH"
|
|
101
|
-
status: "TRIAGED_VALID"
|
|
102
|
-
```
|
|
68
|
+
> [!IMPORTANT]
|
|
69
|
+
> **Decision Rule**: The Judge Agent evaluates findings primarily on **Validity + Severity**.
|
|
70
|
+
> A reviewer's subjective disposition (`STRONG`, `ACCEPTABLE`, `WEAK`) **never overrides evidence**. An `INVALID` finding cannot block delivery, even if the reviewer labeled it `STRONG` or `BLOCKER`.
|