ai-engineering-loop 1.0.21 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/.agents/workflows/ai-engineering-loop.md +7 -3
  2. package/.claude/skills/ai-engineering-loop/SKILL.md +13 -7
  3. package/.gemini/skills/ai-engineering-loop/SKILL.md +13 -7
  4. package/.grok/skills/ai-engineering-loop/SKILL.md +13 -7
  5. package/README.md +26 -3
  6. package/SECURITY.md +23 -0
  7. package/SUPPORT.md +28 -0
  8. package/bin/ai-engineering-loop.js +255 -3
  9. package/core/artifact-gates.md +49 -0
  10. package/evals/README.md +35 -0
  11. package/evals/cases/01-stale-verification.json +17 -0
  12. package/evals/cases/02-vague-verification.json +16 -0
  13. package/evals/cases/03-missing-ac-evidence.json +17 -0
  14. package/evals/cases/04-hallucinated-citation.json +17 -0
  15. package/evals/cases/05-prompt-injection.json +17 -0
  16. package/evals/cases/06-source-secret.json +16 -0
  17. package/evals/cases/07-log-secret.json +16 -0
  18. package/evals/cases/08-symlink-escape.json +16 -0
  19. package/evals/cases/09-authorization-bypass.json +16 -0
  20. package/evals/cases/10-payment-idempotency.json +16 -0
  21. package/evals/cases/11-concurrent-update.json +16 -0
  22. package/evals/cases/12-swallowed-error.json +16 -0
  23. package/evals/cases/13-happy-path-only.json +17 -0
  24. package/evals/cases/14-invalid-finding-axis.json +16 -0
  25. package/evals/cases/15-blocker-without-alternative.json +16 -0
  26. package/evals/cases/16-unsupported-pass.json +17 -0
  27. package/evals/cases/17-soft-standard.json +16 -0
  28. package/evals/cases/18-iteration-ceiling.json +17 -0
  29. package/evals/cases/19-safe-documentation-change.json +17 -0
  30. package/evals/cases/20-valid-boundary-feature.json +18 -0
  31. package/lib/doctor.js +98 -0
  32. package/lib/escalation.js +192 -0
  33. package/lib/evaluation.js +137 -0
  34. package/lib/gates.js +315 -0
  35. package/lib/orchestration.js +5 -1
  36. package/lib/run-state.js +221 -0
  37. package/lib/safe-context.js +191 -0
  38. package/lib/sync-hosts.js +6 -5
  39. package/package.json +12 -9
  40. package/policies/model-escalation.md +41 -0
  41. package/schemas/delivery.schema.json +14 -0
  42. package/schemas/finding-ledger.schema.json +27 -0
  43. package/schemas/goal-contract.schema.json +32 -0
  44. package/schemas/verdict.schema.json +17 -0
  45. package/schemas/verification.schema.json +32 -0
  46. package/docs/antigravity-feasibility.md +0 -92
  47. package/docs/claude-code-feasibility.md +0 -80
  48. package/docs/grok-cli-feasibility.md +0 -99
  49. package/docs/migration-plan.md +0 -70
  50. package/examples/backend-api/payment-idempotency/README.md +0 -33
  51. package/examples/backend-api/payment-idempotency/goal-contract.md +0 -33
  52. package/examples/backend-api/payment-idempotency/judge-verdict.md +0 -28
  53. package/examples/backend-api/payment-idempotency/review-findings.md +0 -50
  54. package/examples/dot/status-display/README.md +0 -23
  55. package/examples/dot/status-display/delivery-report.md +0 -41
  56. package/examples/dot/status-display/goal-contract.md +0 -46
  57. package/examples/dot/status-display/judge-verdict.md +0 -44
  58. package/examples/dot/status-display/review-findings.md +0 -63
  59. package/examples/initialization/README.md +0 -19
  60. package/examples/initialization/discovery-trace.md +0 -63
  61. package/examples/initialization/generated-context.md +0 -110
  62. package/examples/mobile-app/offline-sync-queue/README.md +0 -33
  63. package/examples/mobile-app/offline-sync-queue/goal-contract.md +0 -32
  64. package/examples/mobile-app/offline-sync-queue/judge-verdict.md +0 -27
  65. package/examples/mobile-app/offline-sync-queue/review-findings.md +0 -30
  66. package/scripts/init.sh +0 -106
  67. package/templates/repo-config/adapter.md +0 -11
  68. package/templates/repo-config/adr-readme.md +0 -33
  69. package/templates/repo-config/architecture.md +0 -15
  70. package/templates/repo-config/claude-permissions.json +0 -10
  71. package/templates/repo-config/config.md +0 -11
  72. package/templates/repo-config/conventions.md +0 -16
  73. package/templates/repo-config/glossary.md +0 -18
  74. package/templates/repo-config/lessons.md +0 -8
  75. package/templates/repo-config/verification.md +0 -13
  76. package/templates/repo-config/workflow.md +0 -25
  77. package/tests/capability-selection.test.js +0 -207
  78. package/tests/claimed-vs-reality.test.js +0 -27
  79. package/tests/dot-branch-policy.test.js +0 -65
  80. package/tests/generate-adapter.test.js +0 -80
  81. package/tests/generate-workflow.test.js +0 -180
  82. package/tests/grok-runtime.test.js +0 -201
  83. package/tests/intern-catalog.test.js +0 -72
  84. package/tests/living-context.test.js +0 -61
  85. package/tests/no-company-leak.test.js +0 -82
  86. package/tests/orchestration.test.js +0 -368
  87. package/tests/qa-delivery-template.test.js +0 -90
  88. package/tests/skill-host-compat.test.js +0 -406
  89. package/tests/sync-hosts.test.js +0 -168
@@ -1,368 +0,0 @@
1
- const test = require('node:test');
2
- const assert = require('node:assert');
3
- const {
4
- EXECUTION_MODES,
5
- createCapabilityEvidence,
6
- selectExecutionMode,
7
- buildReviewContextBarrier,
8
- validateVerificationEvidence,
9
- validateFindingLedger,
10
- computeJudgeVerdict
11
- } = require('../lib/orchestration.js');
12
-
13
- // Test A: Independent execution mode selection priority using capability registry
14
- test('A. Independent execution mode selection priority using capability registry', () => {
15
- // Priority 1: Native subagent with full execution proof
16
- assert.strictEqual(
17
- selectExecutionMode({
18
- nativeSubagent: createCapabilityEvidence({
19
- mechanism: 'native-subagent',
20
- classification: 'TRUE_INDEPENDENT_AGENT',
21
- available: true,
22
- executionProven: true,
23
- childConversationId: 'child-101',
24
- executionIdentity: 'exec-101',
25
- modelExecutionProven: true,
26
- independentContextProven: true
27
- }),
28
- sdkAgent: createCapabilityEvidence({
29
- mechanism: 'sdk-agent',
30
- classification: 'ISOLATED_AGENT_INSTANCE',
31
- available: true,
32
- executionProven: true,
33
- childConversationId: 'child-102',
34
- executionIdentity: 'exec-102',
35
- modelExecutionProven: true
36
- })
37
- }).id,
38
- EXECUTION_MODES.TRUE_INDEPENDENT_AGENT.id
39
- );
40
-
41
- // Priority 2: SDK Agent Instance
42
- assert.strictEqual(
43
- selectExecutionMode({
44
- nativeSubagent: createCapabilityEvidence({
45
- mechanism: 'native-subagent',
46
- classification: 'UNAVAILABLE',
47
- available: false
48
- }),
49
- sdkAgent: createCapabilityEvidence({
50
- mechanism: 'sdk-agent',
51
- classification: 'ISOLATED_AGENT_INSTANCE',
52
- available: true,
53
- executionProven: true,
54
- childConversationId: 'child-102',
55
- executionIdentity: 'exec-102',
56
- modelExecutionProven: true
57
- })
58
- }).id,
59
- EXECUTION_MODES.ISOLATED_AGENT_INSTANCE.id
60
- );
61
-
62
- // Priority 3: Headless Subprocess
63
- assert.strictEqual(
64
- selectExecutionMode({
65
- headlessProcessAgent: createCapabilityEvidence({
66
- mechanism: 'cli-agent',
67
- classification: 'FRESH_PROCESS_AGENT',
68
- available: true,
69
- executionProven: true,
70
- executionIdentity: 'pid-8821',
71
- modelExecutionProven: true
72
- })
73
- }).id,
74
- EXECUTION_MODES.FRESH_PROCESS_AGENT.id
75
- );
76
-
77
- // Priority 4: Fallback to Artifact Isolated Review
78
- const fallback = selectExecutionMode({
79
- artifactIsolation: createCapabilityEvidence({
80
- mechanism: 'artifact-barrier',
81
- classification: 'CONTEXT_ISOLATION_ONLY',
82
- available: true
83
- })
84
- });
85
- assert.strictEqual(fallback.id, EXECUTION_MODES.CONTEXT_ISOLATION_ONLY.id);
86
- assert.strictEqual(fallback.isIndependentExecutionProven, false);
87
- assert.strictEqual(fallback.description, 'Isolated review context within the same LLM session; independent agent execution is NOT proven');
88
- });
89
-
90
- // Test B & H: Artifact isolation strictly bounds review context without conversational history
91
- test('B & H. Artifact isolation strictly bounds review context without conversational history', () => {
92
- const goalContract = {
93
- objective: 'Fix race condition in payment queue',
94
- acceptanceCriteria: ['AC-1: Lock payment row before update'],
95
- technicalConstraints: ['Zero schema migrations'],
96
- outOfScope: ['Refund system']
97
- };
98
-
99
- const rawDiff = `
100
- diff --git a/src/pay.ts b/src/pay.ts
101
- + const lock = await db.query('SELECT FOR UPDATE');
102
- `;
103
-
104
- const verificationLogs = {
105
- exitCode: 0,
106
- summary: '10 passed, 0 failed'
107
- };
108
-
109
- const payload = buildReviewContextBarrier({
110
- goalContract,
111
- gitDiff: rawDiff,
112
- verificationLogs,
113
- projectContext: { profile: 'backend-api' }
114
- });
115
-
116
- // Verify only objective fields exist
117
- assert.ok(payload.diffHash);
118
- assert.strictEqual(payload.goalContract.objective, 'Fix race condition in payment queue');
119
- assert.strictEqual(payload.gitDiff.includes('SELECT FOR UPDATE'), true);
120
- assert.strictEqual(payload.verificationLogs.exitCode, 0);
121
-
122
- // Verify conversational scratchpad / thoughts are NOT in payload
123
- assert.strictEqual(payload.makerThoughts, undefined);
124
- assert.strictEqual(payload.chatHistory, undefined);
125
- assert.strictEqual(payload.intermediateDrafts, undefined);
126
- });
127
-
128
- // Test C: Finding Ledger schema validation
129
- test('C. Finding Ledger schema validation', () => {
130
- const validLedger = {
131
- findings: [
132
- {
133
- id: 'DA-1',
134
- topic: 'correctness',
135
- severity: 'BLOCKER',
136
- validity: 'VALID',
137
- disposition: 'STRONG',
138
- location: 'src/pay.ts#L20-L30',
139
- failureScenario: 'Deadlock under concurrent traffic',
140
- evidence: 'Race condition reproduced in load test',
141
- concreteAlternativeDiff: '- lock()\n+ lockWithTimeout()'
142
- }
143
- ]
144
- };
145
-
146
- const check = validateFindingLedger(validLedger);
147
- assert.strictEqual(check.valid, true);
148
-
149
- // Invalid severity test
150
- const invalidLedger = {
151
- findings: [
152
- {
153
- id: 'DA-2',
154
- severity: 'SUPER_CRITICAL', // invalid
155
- validity: 'VALID',
156
- disposition: 'STRONG',
157
- location: 'src/pay.ts#L10',
158
- failureScenario: 'Crash',
159
- evidence: 'Log trace'
160
- }
161
- ]
162
- };
163
- const invalidCheck = validateFindingLedger(invalidLedger);
164
- assert.strictEqual(invalidCheck.valid, false);
165
- });
166
-
167
- // Test E: Verification Evidence Contract
168
- test('E. Verification Evidence Contract strictly enforces execution proof', () => {
169
- // 1. Valid execution evidence
170
- const validEvidence = {
171
- command: 'npm test',
172
- executionIdentity: 'pid-12345',
173
- startTime: '2026-08-25T10:00:00Z',
174
- endTime: '2026-08-25T10:00:05Z',
175
- exitCode: 0,
176
- stdout: 'PASS: 12 tests passed',
177
- stderr: '',
178
- timeoutStatus: 'COMPLETED',
179
- testCounts: { passed: 12, failed: 0, skipped: 0 }
180
- };
181
- assert.strictEqual(validateVerificationEvidence(validEvidence).valid, true);
182
-
183
- // 2. Reject non-zero exit code
184
- const failingEvidence = { ...validEvidence, exitCode: 1 };
185
- assert.strictEqual(validateVerificationEvidence(failingEvidence).valid, false);
186
-
187
- // 3. Reject vague speculative statements without test counts
188
- const vagueEvidence = {
189
- command: 'npm test',
190
- executionIdentity: 'pid-12345',
191
- startTime: '2026-08-25T10:00:00Z',
192
- endTime: '2026-08-25T10:00:05Z',
193
- exitCode: 0,
194
- stdout: 'command was launched in background',
195
- stderr: '',
196
- timeoutStatus: 'COMPLETED',
197
- testCounts: { passed: 0, failed: 0 }
198
- };
199
- assert.strictEqual(validateVerificationEvidence(vagueEvidence).valid, false);
200
- });
201
-
202
- // Test D, F, G: Judge Decision Matrix: VALID BLOCKER forces ITERATE; INVALID does not block delivery
203
- test('D, F, G. Judge Decision Matrix: VALID BLOCKER forces ITERATE; INVALID does not block delivery', () => {
204
- const goalContract = { objective: 'Test' };
205
- const verificationEvidence = {
206
- command: 'npm test',
207
- executionIdentity: 'exec-1',
208
- startTime: '2026-08-25T10:00:00Z',
209
- endTime: '2026-08-25T10:00:02Z',
210
- exitCode: 0,
211
- stdout: '10 passed',
212
- timeoutStatus: 'COMPLETED',
213
- testCounts: { passed: 10, failed: 0 }
214
- };
215
-
216
- // Scenario G: VALID + BLOCKER/HIGH forces ITERATE
217
- const blockerLedger = {
218
- findings: [
219
- {
220
- id: 'DA-1',
221
- severity: 'BLOCKER',
222
- validity: 'VALID',
223
- disposition: 'STRONG',
224
- location: 'src/auth.ts#L10',
225
- failureScenario: 'Auth token unsigned',
226
- evidence: 'JWT verify missing secret',
227
- concreteAlternativeDiff: '+ jwt.verify(token, SECRET)'
228
- }
229
- ]
230
- };
231
-
232
- const verdictBlocker = computeJudgeVerdict({
233
- goalContract,
234
- verificationEvidence,
235
- findingLedger: blockerLedger,
236
- activeIteration: 1
237
- });
238
- assert.strictEqual(verdictBlocker.verdict, 'ITERATE');
239
- assert.strictEqual(verdictBlocker.blockingFindings.length, 1);
240
-
241
- // Scenario F: INVALID is DISMISSED and does NOT block delivery
242
- const invalidLedger = {
243
- findings: [
244
- {
245
- id: 'DA-2',
246
- severity: 'BLOCKER',
247
- validity: 'INVALID', // Hallucinated by reviewer
248
- disposition: 'WEAK',
249
- location: 'src/auth.ts#L20',
250
- failureScenario: 'Reviewer claims function does not exist, but it exists in import',
251
- evidence: 'Import checked and exists'
252
- }
253
- ]
254
- };
255
-
256
- const verdictDismissed = computeJudgeVerdict({
257
- goalContract,
258
- verificationEvidence,
259
- findingLedger: invalidLedger,
260
- activeIteration: 1
261
- });
262
- assert.strictEqual(verdictDismissed.verdict, 'PASS');
263
- assert.strictEqual(verdictDismissed.dismissedFindings.length, 1);
264
-
265
- // Scenario: VALID + LOW/ACCEPTABLE tradeoff passes and documents tradeoff
266
- const tradeoffLedger = {
267
- findings: [
268
- {
269
- id: 'DA-3',
270
- severity: 'LOW',
271
- validity: 'VALID',
272
- disposition: 'ACCEPTABLE',
273
- location: 'src/logger.ts#L5',
274
- failureScenario: 'Log message could be more verbose',
275
- evidence: 'Verbose flag not checked'
276
- }
277
- ]
278
- };
279
-
280
- const verdictTradeoff = computeJudgeVerdict({
281
- goalContract,
282
- verificationEvidence,
283
- findingLedger: tradeoffLedger,
284
- activeIteration: 1
285
- });
286
- assert.strictEqual(verdictTradeoff.verdict, 'PASS');
287
- assert.strictEqual(verdictTradeoff.acceptableTradeoffs.length, 1);
288
- });
289
-
290
- test('Standards axis BLOCKER does not iterate unless hardConvention is true', () => {
291
- const goalContract = { objective: 'Test' };
292
- const verificationEvidence = {
293
- command: 'npm test',
294
- executionIdentity: 'exec-1',
295
- startTime: '2026-08-25T10:00:00Z',
296
- endTime: '2026-08-25T10:00:02Z',
297
- exitCode: 0,
298
- stdout: '10 passed',
299
- timeoutStatus: 'COMPLETED',
300
- testCounts: { passed: 10, failed: 0 }
301
- };
302
-
303
- const smellLedger = {
304
- findings: [
305
- {
306
- id: 'DA-S1',
307
- axis: 'standards',
308
- hardConvention: false,
309
- severity: 'HIGH',
310
- validity: 'VALID',
311
- disposition: 'STRONG',
312
- location: 'src/pay.ts#L4',
313
- failureScenario: 'Duplicated Code across two hunks',
314
- evidence: 'Same lock shape in two functions'
315
- }
316
- ]
317
- };
318
-
319
- const smellVerdict = computeJudgeVerdict({
320
- goalContract,
321
- verificationEvidence,
322
- findingLedger: smellLedger,
323
- activeIteration: 1
324
- });
325
- assert.strictEqual(smellVerdict.verdict, 'PASS');
326
- assert.strictEqual(smellVerdict.acceptableTradeoffs.length, 1);
327
-
328
- const hardLedger = {
329
- findings: [
330
- {
331
- id: 'DA-S2',
332
- axis: 'standards',
333
- hardConvention: true,
334
- severity: 'HIGH',
335
- validity: 'VALID',
336
- disposition: 'STRONG',
337
- location: 'src/pay.ts#L4',
338
- failureScenario: 'conventions.md forbids any',
339
- evidence: 'any used in public seam',
340
- concreteAlternativeDiff: '- x: any\n+ x: unknown'
341
- }
342
- ]
343
- };
344
-
345
- const hardVerdict = computeJudgeVerdict({
346
- goalContract,
347
- verificationEvidence,
348
- findingLedger: hardLedger,
349
- activeIteration: 1
350
- });
351
- assert.strictEqual(hardVerdict.verdict, 'ITERATE');
352
- assert.strictEqual(hardVerdict.blockingFindings.length, 1);
353
- });
354
-
355
- test('Finding ledger rejects unknown axis; missing axis remains valid', () => {
356
- const base = {
357
- id: 'DA-1',
358
- severity: 'LOW',
359
- validity: 'VALID',
360
- disposition: 'ACCEPTABLE',
361
- location: 'src/a.ts#L1',
362
- failureScenario: 'n',
363
- evidence: 'e'
364
- };
365
- assert.strictEqual(validateFindingLedger({ findings: [base] }).valid, true);
366
- assert.strictEqual(validateFindingLedger({ findings: [{ ...base, axis: 'spec' }] }).valid, true);
367
- assert.strictEqual(validateFindingLedger({ findings: [{ ...base, axis: 'style' }] }).valid, false);
368
- });
@@ -1,90 +0,0 @@
1
- 'use strict';
2
-
3
- const test = require('node:test');
4
- const assert = require('node:assert');
5
- const fs = require('fs');
6
- const path = require('path');
7
-
8
- const ROOT = path.join(__dirname, '..');
9
-
10
- function readRepo(rel) {
11
- return fs.readFileSync(path.join(ROOT, rel), 'utf8');
12
- }
13
-
14
- const QA_HEADINGS = [
15
- /## Steps to Reproduce & Testing \(QA\)/,
16
- /### Pre-conditions/,
17
- /### Steps to Reproduce/,
18
- /### How to Test/,
19
- /### Actual Result/,
20
- /### Expected Result/
21
- ];
22
-
23
- const HOSTS = [
24
- '.claude/skills/ai-engineering-loop/SKILL.md',
25
- '.grok/skills/ai-engineering-loop/SKILL.md',
26
- '.gemini/skills/ai-engineering-loop/SKILL.md',
27
- '.agents/workflows/ai-engineering-loop.md'
28
- ];
29
-
30
- test('AC-1 GitHub PR and GitLab MR bodies include the QA reproduce and how-to-test block', () => {
31
- for (const rel of [
32
- 'adapters/github/pull-request.md',
33
- 'adapters/gitlab/merge-request.md'
34
- ]) {
35
- const text = readRepo(rel);
36
- for (const heading of QA_HEADINGS) {
37
- assert.match(text, heading, `${rel} missing ${heading}`);
38
- }
39
- }
40
- });
41
-
42
- test('AC-2 pure feature may mark Actual Result N/A; How to Test stays required', () => {
43
- for (const rel of [
44
- 'adapters/github/pull-request.md',
45
- 'adapters/gitlab/merge-request.md'
46
- ]) {
47
- const text = readRepo(rel);
48
- assert.match(text, /N\/A on a pure feature/, rel);
49
- assert.match(text, /### How to Test/, rel);
50
- assert.match(text, /### Expected Result/, rel);
51
- }
52
- });
53
-
54
- test('AC-3 QA block does not become a dump of diffs, tokens, or machine paths', () => {
55
- for (const rel of [
56
- 'adapters/github/pull-request.md',
57
- 'adapters/gitlab/merge-request.md'
58
- ]) {
59
- const text = readRepo(rel);
60
- assert.match(text, /Do not paste raw diffs, tokens, or local machine paths/, rel);
61
- }
62
- });
63
-
64
- test('AC-4 generic GitLab stays non-DOT; standard does not copy the DOT issue card; DOT issue and MR both have the QA block', () => {
65
- const gitlab = readRepo('adapters/gitlab/merge-request.md');
66
- assert.doesNotMatch(gitlab, /Coreview/);
67
- assert.doesNotMatch(gitlab, /Mattermost/);
68
-
69
- const standard = readRepo('adapters/standard/README.md');
70
- assert.match(standard, /Steps to Reproduce & Testing \(QA\)/);
71
- assert.doesNotMatch(standard, /Ready to Test/);
72
-
73
- const dot = readRepo('adapters/dot/gitlab.md');
74
- const hits = dot.split('## Steps to Reproduce & Testing (QA)').length - 1;
75
- assert.ok(hits >= 2, `DOT gitlab.md needs QA block on issue and MR, got ${hits}`);
76
- for (const heading of QA_HEADINGS) {
77
- assert.match(dot, heading, `DOT gitlab.md missing ${heading}`);
78
- }
79
- });
80
-
81
- test('AC-5 host Stage 8 and DOT Stage 8a require the QA block; bugfix placeholders must not stay empty', () => {
82
- for (const rel of HOSTS) {
83
- const text = readRepo(rel);
84
- assert.match(text, /Steps to Reproduce & Testing \(QA\)/, rel);
85
- }
86
- const delivery = readRepo('adapters/dot/skills/dot-dev-workflow/SKILL.md');
87
- assert.match(delivery, /Steps to Reproduce & Testing \(QA\)/);
88
- assert.match(delivery, /bugfix placeholders must not stay empty|do not leave those placeholders empty/i);
89
- assert.doesNotMatch(delivery, /```mermaid/);
90
- });