ai-engineering-loop 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,409 @@
1
+ /**
2
+ * AI Engineering Loop — Runtime Capability Registry & Orchestration Core
3
+ *
4
+ * Invariants:
5
+ * 1. AI Engineering Loop must NEVER claim independent agent execution without runtime
6
+ * evidence of an actual separate LLM execution.
7
+ * 2. Distinction between CONFIGURATION_SUPPORTED, INVOCATION_AVAILABLE, and EXECUTION_PROVEN:
8
+ * - CONFIGURATION_SUPPORTED: The platform understands subagent configuration (e.g. AGENT.md, subagent: true).
9
+ * - INVOCATION_AVAILABLE: An invocation tool or authenticated CLI is callable in the active runtime.
10
+ * - EXECUTION_PROVEN: A separate child session produced an actual model response with independent context.
11
+ * 3. Artifact isolation is strictly reported as CONTEXT_ISOLATION_ONLY (Independent LLM execution: NOT PROVEN).
12
+ * 4. browser_subagent is browser automation and must NOT be classified as an LLM subagent.
13
+ * 5. agentapi send-message is an IPC communication capability and must NEVER activate agent execution.
14
+ */
15
+
16
+ const crypto = require('crypto');
17
+
18
+ // 1. Standard 5 Execution Modes
19
+ const EXECUTION_MODES = {
20
+ TRUE_INDEPENDENT_AGENT: {
21
+ id: 'TRUE_INDEPENDENT_AGENT',
22
+ label: 'True Independent Agent',
23
+ isIndependentExecutionProven: true,
24
+ description: 'Separate child conversation/process exists, actual LLM execution occurs, and child has independent conversational context'
25
+ },
26
+ ISOLATED_AGENT_INSTANCE: {
27
+ id: 'ISOLATED_AGENT_INSTANCE',
28
+ label: 'Isolated Agent Instance',
29
+ isIndependentExecutionProven: true,
30
+ description: 'Separate conversation or agent instance exists with verified independent model execution'
31
+ },
32
+ FRESH_PROCESS_AGENT: {
33
+ id: 'FRESH_PROCESS_AGENT',
34
+ label: 'Fresh Process Agent',
35
+ isIndependentExecutionProven: true,
36
+ description: 'Separate OS process successfully executes an LLM agent with fresh context'
37
+ },
38
+ CONTEXT_ISOLATION_ONLY: {
39
+ id: 'CONTEXT_ISOLATION_ONLY',
40
+ label: 'Context Isolation Only',
41
+ isIndependentExecutionProven: false,
42
+ description: 'Isolated review context within the same LLM session; independent agent execution is NOT proven'
43
+ },
44
+ UNAVAILABLE: {
45
+ id: 'UNAVAILABLE',
46
+ label: 'Unavailable',
47
+ isIndependentExecutionProven: false,
48
+ description: 'No review execution mechanism is available'
49
+ }
50
+ };
51
+
52
+ /**
53
+ * 2. Standardized Capability Evidence Factory with 3-Stage Lifecycle
54
+ */
55
+ function createCapabilityEvidence({
56
+ mechanism,
57
+ classification,
58
+ configurationSupported = false,
59
+ invocationAvailable = false,
60
+ executionProven = false,
61
+ available = false,
62
+ commandOrApi = null,
63
+ result = null,
64
+ executionIdentity = null,
65
+ conversationId = null,
66
+ parentConversationId = null,
67
+ childConversationId = null,
68
+ childModelResponse = null,
69
+ modelExecutionProven = false,
70
+ independentContextProven = false,
71
+ historyInherited = null,
72
+ isDocumentationOnly = false,
73
+ isBrowserAutomationOnly = false,
74
+ reason = null,
75
+ timestamp = new Date().toISOString()
76
+ }) {
77
+ if (!mechanism || !classification) {
78
+ throw new Error('Capability evidence must include mechanism and classification');
79
+ }
80
+
81
+ const modelExecuted = Boolean(modelExecutionProven || (childModelResponse && executionIdentity));
82
+ const fullyProven = Boolean(executionProven || (modelExecuted && childConversationId && independentContextProven));
83
+
84
+ return {
85
+ mechanism,
86
+ classification,
87
+ configurationSupported: Boolean(configurationSupported),
88
+ invocationAvailable: Boolean(invocationAvailable),
89
+ executionProven: fullyProven,
90
+ available: Boolean(available || fullyProven),
91
+ commandOrApi,
92
+ result,
93
+ executionIdentity,
94
+ conversationId,
95
+ parentConversationId,
96
+ childConversationId,
97
+ childModelResponse,
98
+ modelExecutionProven: modelExecuted,
99
+ independentContextProven: Boolean(independentContextProven),
100
+ historyInherited,
101
+ isDocumentationOnly: Boolean(isDocumentationOnly),
102
+ isBrowserAutomationOnly: Boolean(isBrowserAutomationOnly),
103
+ reason,
104
+ timestamp
105
+ };
106
+ }
107
+
108
+ /**
109
+ * 3. Evaluate whether a capability candidate satisfies execution proof
110
+ */
111
+ function isCapabilityProvenForMode(evidence, targetModeId) {
112
+ if (!evidence) {
113
+ return false;
114
+ }
115
+
116
+ // Reject non-agent capabilities (such as IPC send-message)
117
+ if (
118
+ evidence.classification === 'NOT_AN_AGENT_EXECUTION_CAPABILITY' ||
119
+ evidence.classification === 'IPC_MESSAGE_DISPATCH'
120
+ ) {
121
+ return false;
122
+ }
123
+
124
+ // Reject browser automation tools (browser_subagent is NOT an LLM subagent)
125
+ if (evidence.isBrowserAutomationOnly || evidence.classification === 'BROWSER_AUTOMATION_TOOL') {
126
+ return false;
127
+ }
128
+
129
+ // Reject documentation-only or configuration-only claims
130
+ if (evidence.isDocumentationOnly || (evidence.configurationSupported && !evidence.executionProven)) {
131
+ if (targetModeId !== 'CONTEXT_ISOLATION_ONLY') {
132
+ return false;
133
+ }
134
+ }
135
+
136
+ // Modes requiring true independent LLM execution
137
+ if (
138
+ targetModeId === 'TRUE_INDEPENDENT_AGENT' ||
139
+ targetModeId === 'ISOLATED_AGENT_INSTANCE' ||
140
+ targetModeId === 'FRESH_PROCESS_AGENT'
141
+ ) {
142
+ // Must satisfy all 4 pillars of execution proof:
143
+ const hasChildSession = Boolean(evidence.childConversationId || targetModeId === 'FRESH_PROCESS_AGENT');
144
+ const hasModelResponse = Boolean(evidence.modelExecutionProven || evidence.childModelResponse);
145
+ const hasExecIdentity = Boolean(evidence.executionIdentity);
146
+ const hasCleanContext = Boolean(evidence.independentContextProven && evidence.historyInherited !== true);
147
+
148
+ if (!evidence.executionProven && !(hasChildSession && hasModelResponse && hasExecIdentity && hasCleanContext)) {
149
+ return false;
150
+ }
151
+ }
152
+
153
+ // Context isolation mode requires verified artifact boundary
154
+ if (targetModeId === 'CONTEXT_ISOLATION_ONLY') {
155
+ return evidence.available === true;
156
+ }
157
+
158
+ return true;
159
+ }
160
+
161
+ /**
162
+ * 4. Deterministic Execution Mode Selection
163
+ * Evaluates registry strictly in priority order:
164
+ * 1. TRUE_INDEPENDENT_AGENT
165
+ * 2. ISOLATED_AGENT_INSTANCE
166
+ * 3. FRESH_PROCESS_AGENT
167
+ * 4. CONTEXT_ISOLATION_ONLY
168
+ * 5. UNAVAILABLE
169
+ */
170
+ function selectExecutionMode(capabilityRegistry = {}) {
171
+ const {
172
+ nativeSubagent,
173
+ sdkAgent,
174
+ headlessProcessAgent,
175
+ artifactIsolation
176
+ } = capabilityRegistry;
177
+
178
+ // Priority 1: True Native Independent Sub-Agent
179
+ if (isCapabilityProvenForMode(nativeSubagent, 'TRUE_INDEPENDENT_AGENT')) {
180
+ return EXECUTION_MODES.TRUE_INDEPENDENT_AGENT;
181
+ }
182
+
183
+ // Priority 2: Programmatic SDK Agent Instance
184
+ if (isCapabilityProvenForMode(sdkAgent, 'ISOLATED_AGENT_INSTANCE')) {
185
+ return EXECUTION_MODES.ISOLATED_AGENT_INSTANCE;
186
+ }
187
+
188
+ // Priority 3: Headless Subprocess Agent
189
+ if (isCapabilityProvenForMode(headlessProcessAgent, 'FRESH_PROCESS_AGENT')) {
190
+ return EXECUTION_MODES.FRESH_PROCESS_AGENT;
191
+ }
192
+
193
+ // Priority 4: Guaranteed Fallback - Context Isolation Only
194
+ if (isCapabilityProvenForMode(artifactIsolation, 'CONTEXT_ISOLATION_ONLY')) {
195
+ return EXECUTION_MODES.CONTEXT_ISOLATION_ONLY;
196
+ }
197
+
198
+ // Priority 5: Unavailable
199
+ return EXECUTION_MODES.UNAVAILABLE;
200
+ }
201
+
202
+ /**
203
+ * 5. Build Clean-Slate Artifact Isolation Barrier
204
+ */
205
+ function buildReviewContextBarrier({
206
+ goalContract,
207
+ gitDiff,
208
+ verificationLogs,
209
+ projectContext,
210
+ activeIteration = 1,
211
+ priorFindingSignatures = []
212
+ }) {
213
+ if (!goalContract || !gitDiff) {
214
+ throw new Error('Goal contract and git diff are mandatory for review barrier');
215
+ }
216
+
217
+ return {
218
+ iteration: activeIteration,
219
+ diffHash: crypto.createHash('sha256').update(gitDiff).digest('hex').slice(0, 16),
220
+ goalContract: {
221
+ objective: goalContract.objective,
222
+ acceptanceCriteria: goalContract.acceptanceCriteria || [],
223
+ technicalConstraints: goalContract.technicalConstraints || [],
224
+ outOfScope: goalContract.outOfScope || []
225
+ },
226
+ projectContext: {
227
+ profile: projectContext?.profile || 'standard',
228
+ architecture: projectContext?.architecture || '',
229
+ conventions: projectContext?.conventions || '',
230
+ verificationCommands: projectContext?.verificationCommands || ''
231
+ },
232
+ gitDiff: gitDiff.trim(),
233
+ verificationLogs: {
234
+ exitCode: verificationLogs?.exitCode ?? 0,
235
+ summary: verificationLogs?.summary || 'Deterministic checks passed 100%'
236
+ },
237
+ priorFindingSignatures: [...priorFindingSignatures]
238
+ };
239
+ }
240
+
241
+ /**
242
+ * 6. Validate Verification Evidence Contract
243
+ */
244
+ function validateVerificationEvidence(evidence) {
245
+ if (!evidence) return { valid: false, reason: 'Evidence object missing' };
246
+
247
+ const requiredFields = ['command', 'executionIdentity', 'startTime', 'endTime', 'exitCode', 'stdout', 'timeoutStatus'];
248
+ for (const field of requiredFields) {
249
+ if (evidence[field] === undefined || evidence[field] === null || evidence[field] === '') {
250
+ return { valid: false, reason: `Missing required evidence field: ${field}` };
251
+ }
252
+ }
253
+
254
+ if (evidence.exitCode !== 0) {
255
+ return { valid: false, reason: `Verification failed with non-zero exit code: ${evidence.exitCode}` };
256
+ }
257
+
258
+ if (evidence.timeoutStatus !== 'COMPLETED') {
259
+ return { valid: false, reason: `Verification did not complete cleanly: ${evidence.timeoutStatus}` };
260
+ }
261
+
262
+ const vaguePatterns = ['command was launched', 'test appears to have passed', 'running in background', 'seems green'];
263
+ const outLower = (evidence.stdout + ' ' + (evidence.summary || '')).toLowerCase();
264
+ for (const pattern of vaguePatterns) {
265
+ if (outLower.includes(pattern) && (!evidence.testCounts || evidence.testCounts.passed === 0)) {
266
+ return { valid: false, reason: `Rejected vague or speculative assertion: "${pattern}"` };
267
+ }
268
+ }
269
+
270
+ return { valid: true };
271
+ }
272
+
273
+ /**
274
+ * 7. Validate Finding Ledger Schema
275
+ */
276
+ function validateFindingLedger(ledger) {
277
+ if (!ledger || !Array.isArray(ledger.findings)) {
278
+ return { valid: false, reason: 'Finding ledger must contain an array of findings' };
279
+ }
280
+
281
+ const validSeverities = ['BLOCKER', 'HIGH', 'MEDIUM', 'LOW'];
282
+ const validValidities = ['VALID', 'INVALID'];
283
+ const validDispositions = ['STRONG', 'ACCEPTABLE', 'WEAK'];
284
+
285
+ for (let i = 0; i < ledger.findings.length; i++) {
286
+ const f = ledger.findings[i];
287
+ if (!f.id || !f.location || !f.failureScenario || !f.evidence) {
288
+ return { valid: false, reason: `Finding at index ${i} missing required id, location, failureScenario, or evidence` };
289
+ }
290
+
291
+ if (!validSeverities.includes(f.severity)) {
292
+ return { valid: false, reason: `Invalid severity at index ${i}: ${f.severity}. Must be: ${validSeverities.join(', ')}` };
293
+ }
294
+
295
+ if (!validValidities.includes(f.validity)) {
296
+ return { valid: false, reason: `Invalid validity at index ${i}: ${f.validity}. Must be VALID or INVALID` };
297
+ }
298
+
299
+ if (!validDispositions.includes(f.disposition)) {
300
+ return { valid: false, reason: `Invalid disposition at index ${i}: ${f.disposition}. Must be STRONG, ACCEPTABLE, or WEAK` };
301
+ }
302
+
303
+ if (f.validity === 'VALID' && (f.severity === 'BLOCKER' || f.severity === 'HIGH') && !f.concreteAlternativeDiff) {
304
+ return { valid: false, reason: `VALID ${f.severity} finding ${f.id} must provide concreteAlternativeDiff` };
305
+ }
306
+ }
307
+
308
+ return { valid: true };
309
+ }
310
+
311
+ /**
312
+ * 8. Compute Judge Verdict based primarily on Validity + Severity
313
+ */
314
+ function computeJudgeVerdict({
315
+ goalContract,
316
+ verificationEvidence,
317
+ findingLedger,
318
+ activeIteration = 1,
319
+ maxIterations = 3
320
+ }) {
321
+ const verifCheck = validateVerificationEvidence(verificationEvidence);
322
+ if (!verifCheck.valid) {
323
+ return {
324
+ verdict: 'ITERATE',
325
+ reason: `Deterministic verification failed: ${verifCheck.reason}`,
326
+ action: 'Maker must rerun deterministic test suite and obtain clean exit code 0.'
327
+ };
328
+ }
329
+
330
+ const findings = findingLedger?.findings || [];
331
+ const blockingFindings = [];
332
+ const acceptableFindings = [];
333
+ const dismissedFindings = [];
334
+
335
+ for (const f of findings) {
336
+ if (f.validity === 'INVALID') {
337
+ dismissedFindings.push({
338
+ id: f.id,
339
+ reason: `Dismissed invalid finding: ${f.failureScenario} (Evidence disproved)`
340
+ });
341
+ continue;
342
+ }
343
+
344
+ if (f.severity === 'BLOCKER' || f.severity === 'HIGH') {
345
+ blockingFindings.push(f);
346
+ } else {
347
+ acceptableFindings.push(f);
348
+ }
349
+ }
350
+
351
+ if (blockingFindings.length > 0) {
352
+ if (activeIteration >= maxIterations) {
353
+ return {
354
+ verdict: 'ESCALATE',
355
+ reason: `Max iteration ceiling (${maxIterations}) reached with ${blockingFindings.length} open blocking findings.`,
356
+ blockingFindings,
357
+ action: 'Human escalation triggered with actionable diagnostic ledger.'
358
+ };
359
+ }
360
+
361
+ return {
362
+ verdict: 'ITERATE',
363
+ reason: `${blockingFindings.length} VALID BLOCKER/HIGH findings require code revision.`,
364
+ blockingFindings,
365
+ action: 'Maker must apply concrete alternative diffs and author regression tests.'
366
+ };
367
+ }
368
+
369
+ return {
370
+ verdict: 'PASS',
371
+ reason: `All acceptance criteria verified; 0 open blocking findings; ${dismissedFindings.length} invalid findings dismissed; ${acceptableFindings.length} acceptable tradeoffs documented.`,
372
+ acceptableTradeoffs: acceptableFindings,
373
+ dismissedFindings,
374
+ action: 'Proceed to Context Impact Assessment and Delivery Adapter.'
375
+ };
376
+ }
377
+
378
+ /**
379
+ * 9. Truthful Review Report Generator
380
+ * Enforces mandatory reporting headers and strictly prevents misleading phrasing.
381
+ */
382
+ function formatExecutionReport({ selectedMode, capabilityRegistry = {} }) {
383
+ const isContextOnly = selectedMode.id === EXECUTION_MODES.CONTEXT_ISOLATION_ONLY.id;
384
+
385
+ const nativeSubagentState = capabilityRegistry.nativeSubagent?.invocationAvailable
386
+ ? 'AVAILABLE'
387
+ : 'UNAVAILABLE';
388
+
389
+ const lines = [
390
+ `Execution Mode: ${selectedMode.id}`,
391
+ `Independent LLM Execution: ${selectedMode.isIndependentExecutionProven ? 'PROVEN' : 'NOT PROVEN'}`,
392
+ `Native Subagent Invocation: ${nativeSubagentState}`,
393
+ `Review Method: ${isContextOnly ? 'Clean-Slate Artifact Isolation Barrier' : selectedMode.label}`
394
+ ];
395
+
396
+ return lines.join('\n');
397
+ }
398
+
399
+ module.exports = {
400
+ EXECUTION_MODES,
401
+ createCapabilityEvidence,
402
+ isCapabilityProvenForMode,
403
+ selectExecutionMode,
404
+ buildReviewContextBarrier,
405
+ validateVerificationEvidence,
406
+ validateFindingLedger,
407
+ computeJudgeVerdict,
408
+ formatExecutionReport
409
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai-engineering-loop",
3
- "version": "1.0.1",
3
+ "version": "1.0.3",
4
4
  "description": "A reusable, framework-agnostic AI Engineering Operating System for autonomous coding agents.",
5
5
  "main": "bin/ai-engineering-loop.js",
6
6
  "bin": {
@@ -8,8 +8,7 @@
8
8
  },
9
9
  "scripts": {
10
10
  "init": "node bin/ai-engineering-loop.js",
11
- "prepublishOnly": "node scripts/prepare-npm.js",
12
- "postpublish": "node scripts/restore-github.js"
11
+ "test": "node --test tests/*.test.js"
13
12
  },
14
13
  "repository": {
15
14
  "type": "git",
@@ -10,9 +10,23 @@ An agent may not state that code is functional, bug-free, optimized, or ready fo
10
10
 
11
11
  ---
12
12
 
13
- ## 2. The Evidence Hierarchy
13
+ ## 2. Verification Evidence Contract
14
14
 
15
- Evidence is ranked in order of reliability and truth value:
15
+ A verification `PASS` is strictly invalid without concrete execution evidence. The system categorically rejects vague statements such as *"command was launched"* or *"test appears to have passed"*.
16
+
17
+ ### Mandatory Execution Evidence Properties:
18
+ 1. **`command`**: Exact CLI command executed.
19
+ 2. **`executionIdentity`**: Execution ID / PID / timestamp.
20
+ 3. **`startTime` & `endTime`**: Documenting execution duration.
21
+ 4. **`exitCode`**: Must be `0`.
22
+ 5. **`stdout` & `stderr`**: Raw machine logs captured.
23
+ 6. **`timeoutStatus`**: Must be `"COMPLETED"`.
24
+ 7. **`testCounts`**: Explicit counts of passed, failed, and skipped tests.
25
+ 8. **`assertionEvidence`**: Specific assertion proof matching the active Goal Contract's Acceptance Criteria.
26
+
27
+ ---
28
+
29
+ ## 3. The Evidence Hierarchy
16
30
 
17
31
  ```mermaid
18
32
  flowchart TD
@@ -20,7 +34,7 @@ flowchart TD
20
34
  E2[Level 2: Concrete Codebase Artifacts<br>Actual file contents, git commit history, schema files]
21
35
  E3[Level 3: Adversarial Code Diffs<br>Exact before/after reproduction snippets]
22
36
  E4[Level 4: Theoretical / Analytical Deduction<br>Reasoning through architectural implications]
23
- E5[Level 5: Unsupported Assertion<br>'Looks correct', 'Should work fine' - REJECTED]
37
+ E5[Level 5: Unsupported Assertion<br>'Looks correct', 'Command launched' - REJECTED]
24
38
 
25
39
  E1 --> E2 --> E3 --> E4 -.-> E5
26
40
  ```
@@ -49,22 +63,26 @@ flowchart TD
49
63
 
50
64
  ---
51
65
 
52
- ## 3. Evidence Requirements for Common Agent Claims
66
+ ## 4. Capability Discovery & Orchestration Evidence Rules
67
+
68
+ To prevent misleading claims of multi-agent execution, capability selection must adhere to strict evidence standards:
69
+
70
+ 1. **Documentation is NOT capability proof**: Mentioning `subagent: true` in markdown is not proof of subagent execution.
71
+ 2. **Configuration is NOT execution proof**: A YAML config file is not evidence of a running agent.
72
+ 3. **IPC existence is NOT execution proof**: The existence of an IPC binary (`agentapi`) or successful message dispatch (`send-message`) is an IPC capability, **NOT** an LLM agent execution capability.
73
+ 4. **Conversation creation without model response is NOT an agent**: A conversation ID without an actual captured LLM response is not execution proof.
74
+ 5. **Persona simulation is NEVER an agent**: Switching roles within the same session is self-review, not independent adversarial review.
75
+ 6. **Artifact isolation is strictly labeled**: When review is conducted within the same session via artifact boundaries, it is **strictly reported as `CONTEXT_ISOLATION_ONLY`** with `Independent LLM execution: NOT PROVEN`.
76
+
77
+ ---
78
+
79
+ ## 5. Evidence Requirements for Common Agent Claims
53
80
 
54
81
  | Claim Type | Mandatory Evidence Required | Prohibited Substitute |
55
82
  |---|---|---|
56
83
  | **"Bug is fixed"** | Reproducing test that previously failed now passes with exit code 0. | "The logic was corrected." |
57
84
  | **"No regressions"** | Full test suite execution log showing 0 failures. | "I only touched a single function." |
58
85
  | **"Type safe"** | `tsc --noEmit` / compiler run output with 0 errors. | "I added type annotations." |
59
- | **"Review finding is invalid (Halu)"** | File path & line showing the suggested API does not exist or behavior is intentional. | "I disagree with the reviewer." |
86
+ | **"Review finding is invalid"** | File path & line showing the suggested API does not exist or behavior is intentional. | "I disagree with the reviewer." |
60
87
  | **"Acceptance criteria AC-X met"** | Test function name & assertion specifically targeting AC-X. | "Implemented according to spec." |
61
-
62
- ---
63
-
64
- ## 4. Evidentiary Audit Trail
65
-
66
- Every artifact created during the engineering loop (Goal Contract, Maker Log, Review Findings, Judge Verdict) must maintain an unbroken chain of evidence:
67
-
68
- 1. **Exact Commands Run**: Included in verbatim code blocks.
69
- 2. **Standard Output / Error Logs**: Verbatim output snippets without truncation of error counts.
70
- 3. **Traceable File Links**: Every cited file must use clickable format (`file:///path/to/file#L1-L10`).
88
+ | **"Independent subagent executed"** | Verified parent $\rightarrow$ child session ID + separate process + captured child model response. | "I switched to Devil's Advocate persona." |
@@ -1,102 +1,70 @@
1
1
  # Finding Policy Specification
2
2
 
3
- ## 1. Purpose
3
+ ## 1. Purpose & Core Philosophy
4
4
 
5
- The **Finding Policy** defines the standardized schema, severity definitions, categories, and lifecycle states for all issues identified by review agents (such as the [Devil's Advocate](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/devil-advocate.md) or external review bots).
5
+ The **Finding Policy** defines the standardized schema, severity definitions, validity rules, and lifecycle states for all issues identified by review agents (such as the [Devil's Advocate](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/devil-advocate.md) or external review bots).
6
6
 
7
7
  A standardized finding schema ensures that:
8
- 1. Every criticism is actionable, localized, and backed by evidence.
8
+ 1. Every criticism is actionable, localized, and backed by factual code evidence.
9
9
  2. Review findings can be parsed, hashed, and tracked across autonomous iterations.
10
- 3. The [Judge Agent](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/judge.md) can evaluate findings objectively without subjective ambiguity.
10
+ 3. The [Judge Agent](file:///Users/egagofur/Development/work/ai-engineering-loop/agents/judge.md) can evaluate findings deterministically based on **Validity + Severity** without subjective bias.
11
11
 
12
12
  ---
13
13
 
14
- ## 2. Standardized Finding Schema
14
+ ## 2. Standardized Dual-Axis Finding Schema
15
15
 
16
- Every finding MUST be structured according to the following specification:
16
+ Every finding MUST be structured according to the following dual-axis specification:
17
17
 
18
18
  ```yaml
19
19
  id: "<CATEGORY_PREFIX>-<3_DIGIT_NUMBER>" # e.g. COR-001, SEC-002, PERF-001
20
20
  title: "<Short, descriptive summary of the problem>"
21
- severity: "<CRITICAL | HIGH | MEDIUM | LOW | INFO>"
22
- category: "<Correctness | ErrorHandling | Security | Concurrency | Performance | Maintainability | TestingGaps>"
21
+ topic: "<correctness | error_handling | security | concurrency | performance | maintainability | testing_gaps>"
22
+
23
+ # Axis 1: Factual Validity
24
+ validity: "<VALID | INVALID>" # VALID: Real technical flaw | INVALID: Reviewer hallucination or misunderstanding
25
+
26
+ # Axis 2: Objective Severity
27
+ severity: "<BLOCKER | HIGH | MEDIUM | LOW>"
28
+
29
+ # Reviewer Disposition / Recommendation
30
+ disposition: "<STRONG | ACCEPTABLE | WEAK>"
31
+
23
32
  location:
24
33
  file: "<Relative file path>"
25
34
  startLine: <Integer>
26
35
  endLine: <Integer>
36
+
37
+ acceptanceCriteria: "<AC-1..N impacted, or 'GENERAL_REGRESSION'>"
27
38
  evidence: "<Exact code snippet, command output, or trace exhibiting the flaw>"
28
- problem: "<Detailed technical explanation of what is wrong>"
29
- impact: "<Concrete operational, business, or runtime consequence>"
30
- recommendation: "<Actionable suggestion and concrete code diff showing the fix>"
31
- confidence: "<HIGH | MEDIUM | LOW>"
32
- status: "<NEW | TRIAGED_VALID | TRIAGED_INVALID | TRIAGED_UNCERTAIN | RESOLVED | VERIFIED>"
33
- triageReason: "<Required when status is TRIAGED_INVALID or TRIAGED_UNCERTAIN>"
39
+ failureScenario: "<Concrete step-by-step failure trace describing what breaks at runtime>"
40
+ reproduction: "<Executable test case or sequence demonstrating the bug>"
41
+ concreteAlternativeDiff: |
42
+ ```diff
43
+ - old_flawed_code()
44
+ + new_verified_fix()
45
+ ```
34
46
  ```
35
47
 
36
48
  ---
37
49
 
38
- ## 3. Severity Matrix & Definitions
50
+ ## 3. Severity Matrix & Decision Impact
39
51
 
40
- | Severity Level | Definition | Impact on Build / Loop |
52
+ | Severity Level | Definition | Impact on Verdict |
41
53
  |---|---|---|
42
- | **`CRITICAL` (SEV-1)** | Complete system crash, remote code execution, severe data corruption, authentication bypass, or massive data loss. | **BLOCKING**: Hard block. Must be resolved immediately; cannot pass DoD. |
43
- | **`HIGH` (SEV-2)** | Core business logic defect, unhandled null exception on primary user path, IDOR/authorization gap, or significant performance regression. | **BLOCKING**: Must be resolved before `PASS` verdict. |
44
- | **`MEDIUM` (SEV-3)** | Edge-case logic failure, missing error telemetry, suboptimal query with limited dataset, or unhandled rare boundary condition. | **NON-BLOCKING**: Can pass DoD if documented in backlog, unless directly violating an Acceptance Criterion. |
45
- | **`LOW` (SEV-4)** | Minor maintainability issue, non-critical testing gap, minor code duplication, or naming ambiguity. | **ADVISORY**: Does not block completion. |
46
- | **`INFO` (SEV-5)** | Informational observation, architectural note, or praise for a well-designed pattern. | **INFORMATIONAL**: No action required. |
47
-
48
- ---
49
-
50
- ## 4. Review Categories
51
-
52
- Findings are categorized under one of the 7 standard domains:
53
-
54
- 1. **`Correctness` (`COR`)**: Functional bugs, logic errors, type mismatches, inverted conditionals, timezone bugs, calculation errors.
55
- 2. **`ErrorHandling` (`ERR`)**: Swallowed errors, unhandled rejections, missing fallbacks, crashing async boundaries.
56
- 3. **`Security` (`SEC`)**: Injection, broken auth/access control, secret exposure, sensitive data logging, unsanitized inputs.
57
- 4. **`Concurrency` (`CONC`)**: Race conditions, un-synchronized shared memory, non-atomic database operations, thread safety issues.
58
- 5. **`Performance` (`PERF`)**: N+1 queries, memory leaks, unbounded array loops, unindexed filters, heavy synchronous blocking operations.
59
- 6. **`Maintainability` (`MAINT`)**: High coupling, breaking existing architecture, anti-patterns, dead code, circular dependencies.
60
- 7. **`TestingGaps` (`TEST`)**: Untested boundary conditions, weak/tautological assertions, missing negative test cases.
54
+ | **`BLOCKER`** | System crash, severe data corruption, auth bypass, or direct violation of an explicit Acceptance Criterion. | **BLOCKING**: Forces `ITERATE` verdict. Cannot pass DoD. |
55
+ | **`HIGH`** | Core business logic defect, unhandled null exception on primary user path, or critical performance regression. | **BLOCKING**: Forces `ITERATE` verdict. Must be fixed with regression test. |
56
+ | **`MEDIUM`** | Edge-case logic failure, missing error telemetry, or suboptimal query with limited dataset. | **TRADE-OFF**: Merged if ACs are met; documented as acceptable tradeoff in MR. |
57
+ | **`LOW`** | Minor maintainability issue, non-critical testing gap, minor code duplication, or naming ambiguity. | **TRADE-OFF**: Merged; documented in MR notes. |
61
58
 
62
59
  ---
63
60
 
64
- ## 5. Finding Status & Validity Rules
65
-
66
- Review findings are hypotheses, not absolute truths. Every finding exists in one of the following states:
61
+ ## 4. Validity Rules & Falsification Principle
67
62
 
68
- - **`NEW`**: Initial discovery by reviewer.
69
- - **`TRIAGED_VALID`**: Author and Judge confirm that the defect is real and violates technical or contract requirements. Must be fixed by Maker.
70
- - **`TRIAGED_INVALID` (False Positive / "Halu")**: Author demonstrates with evidence that the reviewer's concern is factually incorrect, based on an API that does not exist, or flags intentional/designed behavior.
71
- - **`TRIAGED_UNCERTAIN`**: Ambiguous requirement where the codebase documentation is silent or contradictory. Triggers escalation.
72
- - **`RESOLVED`**: Maker has applied code fix and passing test suite.
73
- - **`VERIFIED`**: Devil's Advocate and Judge re-evaluated and confirmed the issue is fully solved.
74
-
75
- ---
63
+ Review findings are hypotheses, not absolute truths.
76
64
 
77
- ## 6. Concrete Example
65
+ - **`VALID`**: The defect exists in the code and produces an unhandled failure or contract breach under realistic conditions.
66
+ - **`INVALID` (Dismissed)**: The reviewer made an assumption disproved by the codebase, referenced a non-existent API, or flagged intentional behavior.
78
67
 
79
- ```yaml
80
- id: "COR-001"
81
- title: "Unsafe timezone conversion in attendance clock-in calculation"
82
- severity: "HIGH"
83
- category: "Correctness"
84
- location:
85
- file: "src/server/attendance/utils/calculate-duration.ts"
86
- startLine: 28
87
- endLine: 34
88
- evidence: |
89
- const checkInHour = new Date(record.clockIn).getHours();
90
- problem: |
91
- Using Date.prototype.getHours() extracts the local server timezone hour instead of converting to the employee's designated timezone (e.g. Asia/Jakarta).
92
- impact: |
93
- Employees clocking in between 00:00 and 07:00 UTC will be marked as late or absent depending on the server hosting region.
94
- recommendation: |
95
- Use timezone-aware date parsing with dayjs/date-fns-tz:
96
- ```diff
97
- - const checkInHour = new Date(record.clockIn).getHours();
98
- + const checkInHour = dayjs(record.clockIn).tz(userTimezone).hour();
99
- ```
100
- confidence: "HIGH"
101
- status: "TRIAGED_VALID"
102
- ```
68
+ > [!IMPORTANT]
69
+ > **Decision Rule**: The Judge Agent evaluates findings primarily on **Validity + Severity**.
70
+ > A reviewer's subjective disposition (`STRONG`, `ACCEPTABLE`, `WEAK`) **never overrides evidence**. An `INVALID` finding cannot block delivery, even if the reviewer labeled it `STRONG` or `BLOCKER`.