ai-engineering-loop 1.0.19 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/.agents/workflows/ai-engineering-loop.md +7 -3
  2. package/.claude/skills/ai-engineering-loop/SKILL.md +13 -7
  3. package/.gemini/skills/ai-engineering-loop/SKILL.md +13 -7
  4. package/.grok/skills/ai-engineering-loop/SKILL.md +13 -7
  5. package/README.md +26 -3
  6. package/SECURITY.md +23 -0
  7. package/SUPPORT.md +28 -0
  8. package/adapters/dot/README.md +12 -8
  9. package/adapters/dot/gitlab.md +29 -5
  10. package/adapters/dot/multi-branch.md +47 -68
  11. package/adapters/dot/skills/dot-dev-workflow/SKILL.md +6 -8
  12. package/adapters/github/pull-request.md +15 -0
  13. package/adapters/gitlab/merge-request.md +15 -0
  14. package/adapters/standard/README.md +1 -0
  15. package/bin/ai-engineering-loop.js +255 -3
  16. package/core/artifact-gates.md +49 -0
  17. package/evals/README.md +35 -0
  18. package/evals/cases/01-stale-verification.json +17 -0
  19. package/evals/cases/02-vague-verification.json +16 -0
  20. package/evals/cases/03-missing-ac-evidence.json +17 -0
  21. package/evals/cases/04-hallucinated-citation.json +17 -0
  22. package/evals/cases/05-prompt-injection.json +17 -0
  23. package/evals/cases/06-source-secret.json +16 -0
  24. package/evals/cases/07-log-secret.json +16 -0
  25. package/evals/cases/08-symlink-escape.json +16 -0
  26. package/evals/cases/09-authorization-bypass.json +16 -0
  27. package/evals/cases/10-payment-idempotency.json +16 -0
  28. package/evals/cases/11-concurrent-update.json +16 -0
  29. package/evals/cases/12-swallowed-error.json +16 -0
  30. package/evals/cases/13-happy-path-only.json +17 -0
  31. package/evals/cases/14-invalid-finding-axis.json +16 -0
  32. package/evals/cases/15-blocker-without-alternative.json +16 -0
  33. package/evals/cases/16-unsupported-pass.json +17 -0
  34. package/evals/cases/17-soft-standard.json +16 -0
  35. package/evals/cases/18-iteration-ceiling.json +17 -0
  36. package/evals/cases/19-safe-documentation-change.json +17 -0
  37. package/evals/cases/20-valid-boundary-feature.json +18 -0
  38. package/lib/doctor.js +98 -0
  39. package/lib/escalation.js +192 -0
  40. package/lib/evaluation.js +137 -0
  41. package/lib/gates.js +315 -0
  42. package/lib/orchestration.js +5 -1
  43. package/lib/run-state.js +221 -0
  44. package/lib/safe-context.js +191 -0
  45. package/lib/sync-hosts.js +6 -5
  46. package/package.json +12 -9
  47. package/policies/model-escalation.md +41 -0
  48. package/schemas/delivery.schema.json +14 -0
  49. package/schemas/finding-ledger.schema.json +27 -0
  50. package/schemas/goal-contract.schema.json +32 -0
  51. package/schemas/verdict.schema.json +17 -0
  52. package/schemas/verification.schema.json +32 -0
  53. package/docs/antigravity-feasibility.md +0 -92
  54. package/docs/claude-code-feasibility.md +0 -80
  55. package/docs/grok-cli-feasibility.md +0 -99
  56. package/docs/migration-plan.md +0 -70
  57. package/examples/backend-api/payment-idempotency/README.md +0 -33
  58. package/examples/backend-api/payment-idempotency/goal-contract.md +0 -33
  59. package/examples/backend-api/payment-idempotency/judge-verdict.md +0 -28
  60. package/examples/backend-api/payment-idempotency/review-findings.md +0 -50
  61. package/examples/dot/status-display/README.md +0 -23
  62. package/examples/dot/status-display/delivery-report.md +0 -41
  63. package/examples/dot/status-display/goal-contract.md +0 -46
  64. package/examples/dot/status-display/judge-verdict.md +0 -44
  65. package/examples/dot/status-display/review-findings.md +0 -63
  66. package/examples/initialization/README.md +0 -19
  67. package/examples/initialization/discovery-trace.md +0 -63
  68. package/examples/initialization/generated-context.md +0 -110
  69. package/examples/mobile-app/offline-sync-queue/README.md +0 -33
  70. package/examples/mobile-app/offline-sync-queue/goal-contract.md +0 -32
  71. package/examples/mobile-app/offline-sync-queue/judge-verdict.md +0 -27
  72. package/examples/mobile-app/offline-sync-queue/review-findings.md +0 -30
  73. package/scripts/init.sh +0 -106
  74. package/templates/repo-config/adapter.md +0 -11
  75. package/templates/repo-config/adr-readme.md +0 -33
  76. package/templates/repo-config/architecture.md +0 -15
  77. package/templates/repo-config/claude-permissions.json +0 -10
  78. package/templates/repo-config/config.md +0 -11
  79. package/templates/repo-config/conventions.md +0 -16
  80. package/templates/repo-config/glossary.md +0 -18
  81. package/templates/repo-config/lessons.md +0 -8
  82. package/templates/repo-config/verification.md +0 -13
  83. package/templates/repo-config/workflow.md +0 -25
  84. package/tests/capability-selection.test.js +0 -207
  85. package/tests/claimed-vs-reality.test.js +0 -27
  86. package/tests/generate-adapter.test.js +0 -80
  87. package/tests/generate-workflow.test.js +0 -180
  88. package/tests/grok-runtime.test.js +0 -201
  89. package/tests/intern-catalog.test.js +0 -72
  90. package/tests/living-context.test.js +0 -61
  91. package/tests/no-company-leak.test.js +0 -82
  92. package/tests/orchestration.test.js +0 -368
  93. package/tests/skill-host-compat.test.js +0 -406
  94. package/tests/sync-hosts.test.js +0 -168
@@ -1,368 +0,0 @@
1
- const test = require('node:test');
2
- const assert = require('node:assert');
3
- const {
4
- EXECUTION_MODES,
5
- createCapabilityEvidence,
6
- selectExecutionMode,
7
- buildReviewContextBarrier,
8
- validateVerificationEvidence,
9
- validateFindingLedger,
10
- computeJudgeVerdict
11
- } = require('../lib/orchestration.js');
12
-
13
- // Test A: Independent execution mode selection priority using capability registry
14
- test('A. Independent execution mode selection priority using capability registry', () => {
15
- // Priority 1: Native subagent with full execution proof
16
- assert.strictEqual(
17
- selectExecutionMode({
18
- nativeSubagent: createCapabilityEvidence({
19
- mechanism: 'native-subagent',
20
- classification: 'TRUE_INDEPENDENT_AGENT',
21
- available: true,
22
- executionProven: true,
23
- childConversationId: 'child-101',
24
- executionIdentity: 'exec-101',
25
- modelExecutionProven: true,
26
- independentContextProven: true
27
- }),
28
- sdkAgent: createCapabilityEvidence({
29
- mechanism: 'sdk-agent',
30
- classification: 'ISOLATED_AGENT_INSTANCE',
31
- available: true,
32
- executionProven: true,
33
- childConversationId: 'child-102',
34
- executionIdentity: 'exec-102',
35
- modelExecutionProven: true
36
- })
37
- }).id,
38
- EXECUTION_MODES.TRUE_INDEPENDENT_AGENT.id
39
- );
40
-
41
- // Priority 2: SDK Agent Instance
42
- assert.strictEqual(
43
- selectExecutionMode({
44
- nativeSubagent: createCapabilityEvidence({
45
- mechanism: 'native-subagent',
46
- classification: 'UNAVAILABLE',
47
- available: false
48
- }),
49
- sdkAgent: createCapabilityEvidence({
50
- mechanism: 'sdk-agent',
51
- classification: 'ISOLATED_AGENT_INSTANCE',
52
- available: true,
53
- executionProven: true,
54
- childConversationId: 'child-102',
55
- executionIdentity: 'exec-102',
56
- modelExecutionProven: true
57
- })
58
- }).id,
59
- EXECUTION_MODES.ISOLATED_AGENT_INSTANCE.id
60
- );
61
-
62
- // Priority 3: Headless Subprocess
63
- assert.strictEqual(
64
- selectExecutionMode({
65
- headlessProcessAgent: createCapabilityEvidence({
66
- mechanism: 'cli-agent',
67
- classification: 'FRESH_PROCESS_AGENT',
68
- available: true,
69
- executionProven: true,
70
- executionIdentity: 'pid-8821',
71
- modelExecutionProven: true
72
- })
73
- }).id,
74
- EXECUTION_MODES.FRESH_PROCESS_AGENT.id
75
- );
76
-
77
- // Priority 4: Fallback to Artifact Isolated Review
78
- const fallback = selectExecutionMode({
79
- artifactIsolation: createCapabilityEvidence({
80
- mechanism: 'artifact-barrier',
81
- classification: 'CONTEXT_ISOLATION_ONLY',
82
- available: true
83
- })
84
- });
85
- assert.strictEqual(fallback.id, EXECUTION_MODES.CONTEXT_ISOLATION_ONLY.id);
86
- assert.strictEqual(fallback.isIndependentExecutionProven, false);
87
- assert.strictEqual(fallback.description, 'Isolated review context within the same LLM session; independent agent execution is NOT proven');
88
- });
89
-
90
- // Test B & H: Artifact isolation strictly bounds review context without conversational history
91
- test('B & H. Artifact isolation strictly bounds review context without conversational history', () => {
92
- const goalContract = {
93
- objective: 'Fix race condition in payment queue',
94
- acceptanceCriteria: ['AC-1: Lock payment row before update'],
95
- technicalConstraints: ['Zero schema migrations'],
96
- outOfScope: ['Refund system']
97
- };
98
-
99
- const rawDiff = `
100
- diff --git a/src/pay.ts b/src/pay.ts
101
- + const lock = await db.query('SELECT FOR UPDATE');
102
- `;
103
-
104
- const verificationLogs = {
105
- exitCode: 0,
106
- summary: '10 passed, 0 failed'
107
- };
108
-
109
- const payload = buildReviewContextBarrier({
110
- goalContract,
111
- gitDiff: rawDiff,
112
- verificationLogs,
113
- projectContext: { profile: 'backend-api' }
114
- });
115
-
116
- // Verify only objective fields exist
117
- assert.ok(payload.diffHash);
118
- assert.strictEqual(payload.goalContract.objective, 'Fix race condition in payment queue');
119
- assert.strictEqual(payload.gitDiff.includes('SELECT FOR UPDATE'), true);
120
- assert.strictEqual(payload.verificationLogs.exitCode, 0);
121
-
122
- // Verify conversational scratchpad / thoughts are NOT in payload
123
- assert.strictEqual(payload.makerThoughts, undefined);
124
- assert.strictEqual(payload.chatHistory, undefined);
125
- assert.strictEqual(payload.intermediateDrafts, undefined);
126
- });
127
-
128
- // Test C: Finding Ledger schema validation
129
- test('C. Finding Ledger schema validation', () => {
130
- const validLedger = {
131
- findings: [
132
- {
133
- id: 'DA-1',
134
- topic: 'correctness',
135
- severity: 'BLOCKER',
136
- validity: 'VALID',
137
- disposition: 'STRONG',
138
- location: 'src/pay.ts#L20-L30',
139
- failureScenario: 'Deadlock under concurrent traffic',
140
- evidence: 'Race condition reproduced in load test',
141
- concreteAlternativeDiff: '- lock()\n+ lockWithTimeout()'
142
- }
143
- ]
144
- };
145
-
146
- const check = validateFindingLedger(validLedger);
147
- assert.strictEqual(check.valid, true);
148
-
149
- // Invalid severity test
150
- const invalidLedger = {
151
- findings: [
152
- {
153
- id: 'DA-2',
154
- severity: 'SUPER_CRITICAL', // invalid
155
- validity: 'VALID',
156
- disposition: 'STRONG',
157
- location: 'src/pay.ts#L10',
158
- failureScenario: 'Crash',
159
- evidence: 'Log trace'
160
- }
161
- ]
162
- };
163
- const invalidCheck = validateFindingLedger(invalidLedger);
164
- assert.strictEqual(invalidCheck.valid, false);
165
- });
166
-
167
- // Test E: Verification Evidence Contract
168
- test('E. Verification Evidence Contract strictly enforces execution proof', () => {
169
- // 1. Valid execution evidence
170
- const validEvidence = {
171
- command: 'npm test',
172
- executionIdentity: 'pid-12345',
173
- startTime: '2026-08-25T10:00:00Z',
174
- endTime: '2026-08-25T10:00:05Z',
175
- exitCode: 0,
176
- stdout: 'PASS: 12 tests passed',
177
- stderr: '',
178
- timeoutStatus: 'COMPLETED',
179
- testCounts: { passed: 12, failed: 0, skipped: 0 }
180
- };
181
- assert.strictEqual(validateVerificationEvidence(validEvidence).valid, true);
182
-
183
- // 2. Reject non-zero exit code
184
- const failingEvidence = { ...validEvidence, exitCode: 1 };
185
- assert.strictEqual(validateVerificationEvidence(failingEvidence).valid, false);
186
-
187
- // 3. Reject vague speculative statements without test counts
188
- const vagueEvidence = {
189
- command: 'npm test',
190
- executionIdentity: 'pid-12345',
191
- startTime: '2026-08-25T10:00:00Z',
192
- endTime: '2026-08-25T10:00:05Z',
193
- exitCode: 0,
194
- stdout: 'command was launched in background',
195
- stderr: '',
196
- timeoutStatus: 'COMPLETED',
197
- testCounts: { passed: 0, failed: 0 }
198
- };
199
- assert.strictEqual(validateVerificationEvidence(vagueEvidence).valid, false);
200
- });
201
-
202
- // Test D, F, G: Judge Decision Matrix: VALID BLOCKER forces ITERATE; INVALID does not block delivery
203
- test('D, F, G. Judge Decision Matrix: VALID BLOCKER forces ITERATE; INVALID does not block delivery', () => {
204
- const goalContract = { objective: 'Test' };
205
- const verificationEvidence = {
206
- command: 'npm test',
207
- executionIdentity: 'exec-1',
208
- startTime: '2026-08-25T10:00:00Z',
209
- endTime: '2026-08-25T10:00:02Z',
210
- exitCode: 0,
211
- stdout: '10 passed',
212
- timeoutStatus: 'COMPLETED',
213
- testCounts: { passed: 10, failed: 0 }
214
- };
215
-
216
- // Scenario G: VALID + BLOCKER/HIGH forces ITERATE
217
- const blockerLedger = {
218
- findings: [
219
- {
220
- id: 'DA-1',
221
- severity: 'BLOCKER',
222
- validity: 'VALID',
223
- disposition: 'STRONG',
224
- location: 'src/auth.ts#L10',
225
- failureScenario: 'Auth token unsigned',
226
- evidence: 'JWT verify missing secret',
227
- concreteAlternativeDiff: '+ jwt.verify(token, SECRET)'
228
- }
229
- ]
230
- };
231
-
232
- const verdictBlocker = computeJudgeVerdict({
233
- goalContract,
234
- verificationEvidence,
235
- findingLedger: blockerLedger,
236
- activeIteration: 1
237
- });
238
- assert.strictEqual(verdictBlocker.verdict, 'ITERATE');
239
- assert.strictEqual(verdictBlocker.blockingFindings.length, 1);
240
-
241
- // Scenario F: INVALID is DISMISSED and does NOT block delivery
242
- const invalidLedger = {
243
- findings: [
244
- {
245
- id: 'DA-2',
246
- severity: 'BLOCKER',
247
- validity: 'INVALID', // Hallucinated by reviewer
248
- disposition: 'WEAK',
249
- location: 'src/auth.ts#L20',
250
- failureScenario: 'Reviewer claims function does not exist, but it exists in import',
251
- evidence: 'Import checked and exists'
252
- }
253
- ]
254
- };
255
-
256
- const verdictDismissed = computeJudgeVerdict({
257
- goalContract,
258
- verificationEvidence,
259
- findingLedger: invalidLedger,
260
- activeIteration: 1
261
- });
262
- assert.strictEqual(verdictDismissed.verdict, 'PASS');
263
- assert.strictEqual(verdictDismissed.dismissedFindings.length, 1);
264
-
265
- // Scenario: VALID + LOW/ACCEPTABLE tradeoff passes and documents tradeoff
266
- const tradeoffLedger = {
267
- findings: [
268
- {
269
- id: 'DA-3',
270
- severity: 'LOW',
271
- validity: 'VALID',
272
- disposition: 'ACCEPTABLE',
273
- location: 'src/logger.ts#L5',
274
- failureScenario: 'Log message could be more verbose',
275
- evidence: 'Verbose flag not checked'
276
- }
277
- ]
278
- };
279
-
280
- const verdictTradeoff = computeJudgeVerdict({
281
- goalContract,
282
- verificationEvidence,
283
- findingLedger: tradeoffLedger,
284
- activeIteration: 1
285
- });
286
- assert.strictEqual(verdictTradeoff.verdict, 'PASS');
287
- assert.strictEqual(verdictTradeoff.acceptableTradeoffs.length, 1);
288
- });
289
-
290
- test('Standards axis BLOCKER does not iterate unless hardConvention is true', () => {
291
- const goalContract = { objective: 'Test' };
292
- const verificationEvidence = {
293
- command: 'npm test',
294
- executionIdentity: 'exec-1',
295
- startTime: '2026-08-25T10:00:00Z',
296
- endTime: '2026-08-25T10:00:02Z',
297
- exitCode: 0,
298
- stdout: '10 passed',
299
- timeoutStatus: 'COMPLETED',
300
- testCounts: { passed: 10, failed: 0 }
301
- };
302
-
303
- const smellLedger = {
304
- findings: [
305
- {
306
- id: 'DA-S1',
307
- axis: 'standards',
308
- hardConvention: false,
309
- severity: 'HIGH',
310
- validity: 'VALID',
311
- disposition: 'STRONG',
312
- location: 'src/pay.ts#L4',
313
- failureScenario: 'Duplicated Code across two hunks',
314
- evidence: 'Same lock shape in two functions'
315
- }
316
- ]
317
- };
318
-
319
- const smellVerdict = computeJudgeVerdict({
320
- goalContract,
321
- verificationEvidence,
322
- findingLedger: smellLedger,
323
- activeIteration: 1
324
- });
325
- assert.strictEqual(smellVerdict.verdict, 'PASS');
326
- assert.strictEqual(smellVerdict.acceptableTradeoffs.length, 1);
327
-
328
- const hardLedger = {
329
- findings: [
330
- {
331
- id: 'DA-S2',
332
- axis: 'standards',
333
- hardConvention: true,
334
- severity: 'HIGH',
335
- validity: 'VALID',
336
- disposition: 'STRONG',
337
- location: 'src/pay.ts#L4',
338
- failureScenario: 'conventions.md forbids any',
339
- evidence: 'any used in public seam',
340
- concreteAlternativeDiff: '- x: any\n+ x: unknown'
341
- }
342
- ]
343
- };
344
-
345
- const hardVerdict = computeJudgeVerdict({
346
- goalContract,
347
- verificationEvidence,
348
- findingLedger: hardLedger,
349
- activeIteration: 1
350
- });
351
- assert.strictEqual(hardVerdict.verdict, 'ITERATE');
352
- assert.strictEqual(hardVerdict.blockingFindings.length, 1);
353
- });
354
-
355
- test('Finding ledger rejects unknown axis; missing axis remains valid', () => {
356
- const base = {
357
- id: 'DA-1',
358
- severity: 'LOW',
359
- validity: 'VALID',
360
- disposition: 'ACCEPTABLE',
361
- location: 'src/a.ts#L1',
362
- failureScenario: 'n',
363
- evidence: 'e'
364
- };
365
- assert.strictEqual(validateFindingLedger({ findings: [base] }).valid, true);
366
- assert.strictEqual(validateFindingLedger({ findings: [{ ...base, axis: 'spec' }] }).valid, true);
367
- assert.strictEqual(validateFindingLedger({ findings: [{ ...base, axis: 'style' }] }).valid, false);
368
- });