@mmnto/totem 1.14.10 → 1.14.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compile-lesson.d.ts +76 -1
- package/dist/compile-lesson.d.ts.map +1 -1
- package/dist/compile-lesson.js +481 -53
- package/dist/compile-lesson.js.map +1 -1
- package/dist/compile-lesson.test.js +756 -8
- package/dist/compile-lesson.test.js.map +1 -1
- package/dist/compiler-schema.d.ts +185 -9
- package/dist/compiler-schema.d.ts.map +1 -1
- package/dist/compiler-schema.js +95 -10
- package/dist/compiler-schema.js.map +1 -1
- package/dist/compiler.d.ts +11 -3
- package/dist/compiler.d.ts.map +1 -1
- package/dist/compiler.js +24 -4
- package/dist/compiler.js.map +1 -1
- package/dist/compiler.test.js +162 -22
- package/dist/compiler.test.js.map +1 -1
- package/dist/config-schema.d.ts +86 -0
- package/dist/config-schema.d.ts.map +1 -1
- package/dist/config-schema.js +54 -0
- package/dist/config-schema.js.map +1 -1
- package/dist/config-schema.test.js +137 -1
- package/dist/config-schema.test.js.map +1 -1
- package/dist/index.d.ts +8 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/ledger.d.ts +10 -0
- package/dist/ledger.d.ts.map +1 -1
- package/dist/ledger.js +8 -0
- package/dist/ledger.js.map +1 -1
- package/dist/lesson-pattern.d.ts.map +1 -1
- package/dist/lesson-pattern.js +6 -9
- package/dist/lesson-pattern.js.map +1 -1
- package/dist/pack-merge.d.ts +73 -0
- package/dist/pack-merge.d.ts.map +1 -0
- package/dist/pack-merge.js +117 -0
- package/dist/pack-merge.js.map +1 -0
- package/dist/pack-merge.test.d.ts +2 -0
- package/dist/pack-merge.test.d.ts.map +1 -0
- package/dist/pack-merge.test.js +238 -0
- package/dist/pack-merge.test.js.map +1 -0
- package/dist/regex-utils.d.ts +5 -0
- package/dist/regex-utils.d.ts.map +1 -1
- package/dist/regex-utils.js +6 -1
- package/dist/regex-utils.js.map +1 -1
- package/dist/regex-utils.test.js +15 -2
- package/dist/regex-utils.test.js.map +1 -1
- package/dist/rule-engine.d.ts.map +1 -1
- package/dist/rule-engine.js +3 -0
- package/dist/rule-engine.js.map +1 -1
- package/dist/rule-engine.test.js +29 -0
- package/dist/rule-engine.test.js.map +1 -1
- package/dist/rule-metrics.d.ts +40 -0
- package/dist/rule-metrics.d.ts.map +1 -1
- package/dist/rule-metrics.js +28 -0
- package/dist/rule-metrics.js.map +1 -1
- package/dist/rule-metrics.test.js +104 -3
- package/dist/rule-metrics.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
-
import { buildCompiledRule, buildManualRule, compileLesson, validateAstGrepPattern, verifyRuleExamples, } from './compile-lesson.js';
|
|
2
|
+
import { buildCompiledRule, buildManualRule, compileLesson, isSecurityContext, validateAstGrepPattern, verifyRuleExamples, } from './compile-lesson.js';
|
|
3
3
|
import { CompilerOutputSchema } from './compiler-schema.js';
|
|
4
4
|
// ─── Helpers ────────────────────────────────────────
|
|
5
5
|
const lesson = {
|
|
@@ -767,6 +767,13 @@ describe('compileLesson', () => {
|
|
|
767
767
|
});
|
|
768
768
|
// ─── Smoke gate on Pipeline 2 (mmnto/totem#1408) ──
|
|
769
769
|
it('Pipeline 2 rejects a rule whose LLM output omits badExample', async () => {
|
|
770
|
+
// Post-ADR-088 (mmnto-ai/totem#1479): missing badExample short-circuits
|
|
771
|
+
// to `status: 'skipped'` with reasonCode `'missing-badexample'` rather
|
|
772
|
+
// than `'failed'`. The Layer 4 explicit-failure contract requires every
|
|
773
|
+
// skipped lesson to carry a machine-readable reason; emitting `'failed'`
|
|
774
|
+
// would drop the entry without a trace. No retry because the compiler
|
|
775
|
+
// system prompt already requires the field (mmnto-ai/totem#1409) and
|
|
776
|
+
// retrying won't teach the LLM to emit a field it just omitted.
|
|
770
777
|
const deps = {
|
|
771
778
|
parseCompilerResponse: vi.fn().mockReturnValue({
|
|
772
779
|
compilable: true,
|
|
@@ -780,7 +787,12 @@ describe('compileLesson', () => {
|
|
|
780
787
|
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
781
788
|
};
|
|
782
789
|
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
783
|
-
expect(result.status).toBe('
|
|
790
|
+
expect(result.status).toBe('skipped');
|
|
791
|
+
if (result.status === 'skipped') {
|
|
792
|
+
expect(result.reasonCode).toBe('missing-badexample');
|
|
793
|
+
expect(result.reason).toContain('missing badExample');
|
|
794
|
+
}
|
|
795
|
+
expect(deps.runOrchestrator).toHaveBeenCalledTimes(1);
|
|
784
796
|
expect(deps.callbacks.onWarn).toHaveBeenCalledWith(lesson.heading, expect.stringContaining('smoke gate'));
|
|
785
797
|
});
|
|
786
798
|
it('Pipeline 2 accepts a rule whose LLM output supplies a matching badExample', async () => {
|
|
@@ -900,7 +912,11 @@ describe('compileLesson', () => {
|
|
|
900
912
|
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
901
913
|
expect(result.status).toBe('noop');
|
|
902
914
|
});
|
|
903
|
-
it('returns
|
|
915
|
+
it('returns skipped with pattern-syntax-invalid when parseCompilerResponse returns null', async () => {
|
|
916
|
+
// mmnto-ai/totem#1481: the Pipeline 2 parse-failure exit route was
|
|
917
|
+
// upgraded from 'failed' to 'skipped' with a machine-readable
|
|
918
|
+
// reasonCode so ADR-088 Layer 4 telemetry sees every LLM-output
|
|
919
|
+
// parse error rather than losing them to the 'failed' bucket.
|
|
904
920
|
const deps = {
|
|
905
921
|
parseCompilerResponse: vi.fn().mockReturnValue(null),
|
|
906
922
|
runOrchestrator: vi.fn().mockResolvedValue('bad response'),
|
|
@@ -908,7 +924,10 @@ describe('compileLesson', () => {
|
|
|
908
924
|
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
909
925
|
};
|
|
910
926
|
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
911
|
-
expect(result.status).toBe('
|
|
927
|
+
expect(result.status).toBe('skipped');
|
|
928
|
+
if (result.status === 'skipped') {
|
|
929
|
+
expect(result.reasonCode).toBe('pattern-syntax-invalid');
|
|
930
|
+
}
|
|
912
931
|
expect(deps.callbacks.onWarn).toHaveBeenCalled();
|
|
913
932
|
});
|
|
914
933
|
it('returns skipped for non-compilable lessons', async () => {
|
|
@@ -1216,7 +1235,10 @@ describe('compileLesson Pipeline 3 (Bad/Good snippets)', () => {
|
|
|
1216
1235
|
// Pattern matches neither Bad nor Good - smoke gate catches this
|
|
1217
1236
|
// before the old self-verification step ever runs. Pre-#1408 the
|
|
1218
1237
|
// test asserted on the self-verification message; post-#1408 the
|
|
1219
|
-
// smoke gate is the earlier (and stricter) filter.
|
|
1238
|
+
// smoke gate is the earlier (and stricter) filter. mmnto-ai/totem#1481
|
|
1239
|
+
// further promotes the smoke-gate zero-match branch from 'failed'
|
|
1240
|
+
// to 'skipped' with reasonCode 'pattern-syntax-invalid' so Layer 4
|
|
1241
|
+
// telemetry gets a machine-readable entry.
|
|
1220
1242
|
pattern: 'this_matches_nothing_at_all',
|
|
1221
1243
|
message: 'Wrong pattern',
|
|
1222
1244
|
engine: 'regex',
|
|
@@ -1226,7 +1248,11 @@ describe('compileLesson Pipeline 3 (Bad/Good snippets)', () => {
|
|
|
1226
1248
|
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1227
1249
|
};
|
|
1228
1250
|
const result = await compileLesson(pipeline3Lesson, 'system prompt', deps);
|
|
1229
|
-
expect(result.status).toBe('
|
|
1251
|
+
expect(result.status).toBe('skipped');
|
|
1252
|
+
if (result.status === 'skipped') {
|
|
1253
|
+
expect(result.reasonCode).toBe('pattern-zero-match');
|
|
1254
|
+
expect(result.reason).toContain('smoke gate');
|
|
1255
|
+
}
|
|
1230
1256
|
expect(deps.callbacks.onWarn).toHaveBeenCalledWith(pipeline3Lesson.heading, expect.stringContaining('smoke gate'));
|
|
1231
1257
|
});
|
|
1232
1258
|
it('passes self-verification when pattern matches Bad but not Good', async () => {
|
|
@@ -1290,7 +1316,9 @@ describe('compileLesson Pipeline 3 (Bad/Good snippets)', () => {
|
|
|
1290
1316
|
const result = await compileLesson(pipeline3Lesson, 'system prompt', deps);
|
|
1291
1317
|
expect(result.status).toBe('noop');
|
|
1292
1318
|
});
|
|
1293
|
-
it('returns
|
|
1319
|
+
it('returns skipped with pattern-syntax-invalid when Pipeline 3 LLM response cannot be parsed', async () => {
|
|
1320
|
+
// mmnto-ai/totem#1481: Pipeline 3 parse-failure now lands in the ledger
|
|
1321
|
+
// with a machine-readable reasonCode per ADR-088 Layer 4.
|
|
1294
1322
|
const deps = {
|
|
1295
1323
|
parseCompilerResponse: vi.fn().mockReturnValue(null),
|
|
1296
1324
|
runOrchestrator: vi.fn().mockResolvedValue('bad response'),
|
|
@@ -1298,8 +1326,728 @@ describe('compileLesson Pipeline 3 (Bad/Good snippets)', () => {
|
|
|
1298
1326
|
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1299
1327
|
};
|
|
1300
1328
|
const result = await compileLesson(pipeline3Lesson, 'system prompt', deps);
|
|
1301
|
-
expect(result.status).toBe('
|
|
1329
|
+
expect(result.status).toBe('skipped');
|
|
1330
|
+
if (result.status === 'skipped') {
|
|
1331
|
+
expect(result.reasonCode).toBe('pattern-syntax-invalid');
|
|
1332
|
+
}
|
|
1302
1333
|
expect(deps.callbacks.onWarn).toHaveBeenCalledWith(pipeline3Lesson.heading, expect.stringContaining('Pipeline 3'));
|
|
1303
1334
|
});
|
|
1304
1335
|
});
|
|
1336
|
+
// ─── Pipeline 2 Layer 3 verify-retry loop (ADR-088, mmnto-ai/totem#1479) ──
|
|
1337
|
+
describe('compileLesson Pipeline 2 verify-retry', () => {
|
|
1338
|
+
const lesson = {
|
|
1339
|
+
index: 0,
|
|
1340
|
+
heading: 'No console.log in production code',
|
|
1341
|
+
body: 'Do not use console.log in production.',
|
|
1342
|
+
hash: 'h-retry',
|
|
1343
|
+
};
|
|
1344
|
+
it('retries after smoke-gate zero-match and succeeds when a later attempt produces a matching pattern', async () => {
|
|
1345
|
+
// Attempt 1 returns a pattern that does not match the badExample.
|
|
1346
|
+
// Attempt 2 returns a pattern that does. Expected outcome: compiled,
|
|
1347
|
+
// two orchestrator calls, retry feedback threaded into attempt 2.
|
|
1348
|
+
const parseMock = vi
|
|
1349
|
+
.fn()
|
|
1350
|
+
.mockReturnValueOnce({
|
|
1351
|
+
compilable: true,
|
|
1352
|
+
pattern: 'never_matches_xyz',
|
|
1353
|
+
message: 'No console.log',
|
|
1354
|
+
engine: 'regex',
|
|
1355
|
+
badExample: 'console.log("debug")',
|
|
1356
|
+
})
|
|
1357
|
+
.mockReturnValueOnce({
|
|
1358
|
+
compilable: true,
|
|
1359
|
+
pattern: 'console\\.log',
|
|
1360
|
+
message: 'No console.log',
|
|
1361
|
+
engine: 'regex',
|
|
1362
|
+
badExample: 'console.log("debug")',
|
|
1363
|
+
});
|
|
1364
|
+
const orchestratorMock = vi
|
|
1365
|
+
.fn()
|
|
1366
|
+
.mockResolvedValueOnce('attempt-1')
|
|
1367
|
+
.mockResolvedValueOnce('attempt-2');
|
|
1368
|
+
const deps = {
|
|
1369
|
+
parseCompilerResponse: parseMock,
|
|
1370
|
+
runOrchestrator: orchestratorMock,
|
|
1371
|
+
existingByHash: new Map(),
|
|
1372
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1373
|
+
};
|
|
1374
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1375
|
+
expect(result.status).toBe('compiled');
|
|
1376
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(2);
|
|
1377
|
+
// The retry-directive must appear in the user prompt of attempt 2 and
|
|
1378
|
+
// must not appear in attempt 1. The directive carries the failed
|
|
1379
|
+
// pattern and the badExample so the LLM can correct its output.
|
|
1380
|
+
const userPrompt1 = orchestratorMock.mock.calls[0][0];
|
|
1381
|
+
const userPrompt2 = orchestratorMock.mock.calls[1][0];
|
|
1382
|
+
expect(userPrompt1).not.toContain('Previous Attempt Failed Verification');
|
|
1383
|
+
expect(userPrompt2).toContain('Previous Attempt Failed Verification');
|
|
1384
|
+
expect(userPrompt2).toContain('never_matches_xyz');
|
|
1385
|
+
expect(userPrompt2).toContain('console.log("debug")');
|
|
1386
|
+
});
|
|
1387
|
+
it('returns skipped with reasonCode verify-retry-exhausted after MAX_VERIFY_ATTEMPTS failures', async () => {
|
|
1388
|
+
// Every attempt returns a pattern that does not match the badExample.
|
|
1389
|
+
// Expected outcome: skipped, reasonCode 'verify-retry-exhausted',
|
|
1390
|
+
// exactly 3 orchestrator calls.
|
|
1391
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1392
|
+
compilable: true,
|
|
1393
|
+
pattern: 'never_matches_xyz',
|
|
1394
|
+
message: 'No console.log',
|
|
1395
|
+
engine: 'regex',
|
|
1396
|
+
badExample: 'console.log("debug")',
|
|
1397
|
+
});
|
|
1398
|
+
const orchestratorMock = vi.fn().mockResolvedValue('always-bad');
|
|
1399
|
+
const deps = {
|
|
1400
|
+
parseCompilerResponse: parseMock,
|
|
1401
|
+
runOrchestrator: orchestratorMock,
|
|
1402
|
+
existingByHash: new Map(),
|
|
1403
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1404
|
+
};
|
|
1405
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1406
|
+
expect(result.status).toBe('skipped');
|
|
1407
|
+
if (result.status === 'skipped') {
|
|
1408
|
+
expect(result.reasonCode).toBe('verify-retry-exhausted');
|
|
1409
|
+
expect(result.reason).toContain('Verify retry exhausted after 3 attempts');
|
|
1410
|
+
}
|
|
1411
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(3);
|
|
1412
|
+
});
|
|
1413
|
+
it('rejects a security-context rule without retry when verify fails', async () => {
|
|
1414
|
+
// securityContext: true + zero-match on attempt 1. Zero tolerance per
|
|
1415
|
+
// ADR-088 Decision 3 means no retry. Expected outcome: skipped,
|
|
1416
|
+
// reasonCode 'security-rule-rejected', exactly 1 orchestrator call.
|
|
1417
|
+
//
|
|
1418
|
+
// mmnto-ai/totem#1480 added an upfront Example-Hit short-circuit for
|
|
1419
|
+
// security context, so this test uses a lesson body that includes a
|
|
1420
|
+
// non-empty Example Hit block. That keeps the test exercising the
|
|
1421
|
+
// verify-path security branch, not the new no-example-hit short-circuit.
|
|
1422
|
+
const securityLesson = {
|
|
1423
|
+
index: 0,
|
|
1424
|
+
heading: 'No shell injection in spawn',
|
|
1425
|
+
body: [
|
|
1426
|
+
'Do not pass untrusted input to spawn with shell: true.',
|
|
1427
|
+
'**Example Hit:** spawn("sh", ["-c", userInput])',
|
|
1428
|
+
].join('\n'),
|
|
1429
|
+
hash: 'h-security-verify',
|
|
1430
|
+
};
|
|
1431
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1432
|
+
compilable: true,
|
|
1433
|
+
pattern: 'never_matches_xyz',
|
|
1434
|
+
message: 'Security rule that failed to verify',
|
|
1435
|
+
engine: 'regex',
|
|
1436
|
+
badExample: 'spawn("sh", ["-c", userInput])',
|
|
1437
|
+
});
|
|
1438
|
+
const orchestratorMock = vi.fn().mockResolvedValue('security-bad');
|
|
1439
|
+
const deps = {
|
|
1440
|
+
parseCompilerResponse: parseMock,
|
|
1441
|
+
runOrchestrator: orchestratorMock,
|
|
1442
|
+
existingByHash: new Map(),
|
|
1443
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1444
|
+
securityContext: true,
|
|
1445
|
+
};
|
|
1446
|
+
const result = await compileLesson(securityLesson, 'system prompt', deps);
|
|
1447
|
+
expect(result.status).toBe('skipped');
|
|
1448
|
+
if (result.status === 'skipped') {
|
|
1449
|
+
expect(result.reasonCode).toBe('security-rule-rejected');
|
|
1450
|
+
expect(result.reason).toContain('zero matches');
|
|
1451
|
+
}
|
|
1452
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(1);
|
|
1453
|
+
expect(deps.callbacks.onWarn).toHaveBeenCalledWith(securityLesson.heading, expect.stringContaining('Security rule rejected on verify failure'));
|
|
1454
|
+
});
|
|
1455
|
+
it('does not retry when the smoke-gate rejection is missing badExample (short-circuit to skipped)', async () => {
|
|
1456
|
+
// Missing badExample is a structural-output failure. Retrying cannot
|
|
1457
|
+
// teach the LLM to emit a field it just omitted. Expected outcome:
|
|
1458
|
+
// skipped, reasonCode 'missing-badexample', exactly 1 orchestrator
|
|
1459
|
+
// call. Same contract exercised by the pre-retry test above but
|
|
1460
|
+
// explicitly scoped to the retry-loop branch.
|
|
1461
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1462
|
+
compilable: true,
|
|
1463
|
+
pattern: 'console\\.log',
|
|
1464
|
+
message: 'No console.log',
|
|
1465
|
+
engine: 'regex',
|
|
1466
|
+
// no badExample
|
|
1467
|
+
});
|
|
1468
|
+
const orchestratorMock = vi.fn().mockResolvedValue('no-badexample');
|
|
1469
|
+
const deps = {
|
|
1470
|
+
parseCompilerResponse: parseMock,
|
|
1471
|
+
runOrchestrator: orchestratorMock,
|
|
1472
|
+
existingByHash: new Map(),
|
|
1473
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1474
|
+
};
|
|
1475
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1476
|
+
expect(result.status).toBe('skipped');
|
|
1477
|
+
if (result.status === 'skipped') {
|
|
1478
|
+
expect(result.reasonCode).toBe('missing-badexample');
|
|
1479
|
+
}
|
|
1480
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(1);
|
|
1481
|
+
});
|
|
1482
|
+
it('propagates non-compilable LLM verdict with reasonCode out-of-scope', async () => {
|
|
1483
|
+
// Conceptual/architectural lessons that the LLM classifies as
|
|
1484
|
+
// non-compilable. ADR-088 Layer 4 requires a machine-readable reason.
|
|
1485
|
+
// mmnto-ai/totem#1481 renamed `'non-compilable'` to `'out-of-scope'`
|
|
1486
|
+
// to align the internal reason code with the persisted enum.
|
|
1487
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1488
|
+
compilable: false,
|
|
1489
|
+
reason: 'Architectural principle, not a pattern.',
|
|
1490
|
+
});
|
|
1491
|
+
const orchestratorMock = vi.fn().mockResolvedValue('{"compilable": false}');
|
|
1492
|
+
const deps = {
|
|
1493
|
+
parseCompilerResponse: parseMock,
|
|
1494
|
+
runOrchestrator: orchestratorMock,
|
|
1495
|
+
existingByHash: new Map(),
|
|
1496
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1497
|
+
};
|
|
1498
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1499
|
+
expect(result.status).toBe('skipped');
|
|
1500
|
+
if (result.status === 'skipped') {
|
|
1501
|
+
expect(result.reasonCode).toBe('out-of-scope');
|
|
1502
|
+
expect(result.reason).toBe('Architectural principle, not a pattern.');
|
|
1503
|
+
}
|
|
1504
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(1);
|
|
1505
|
+
});
|
|
1506
|
+
it('retries when Example Hit/Miss verification fails and succeeds on a later attempt', async () => {
|
|
1507
|
+
// ADR-088 AC (mmnto-ai/totem#1479): "verifies every LLM-generated pattern
|
|
1508
|
+
// against the lesson's Example Hit block. Zero-match triggers a retry."
|
|
1509
|
+
// Attempt 1 produces a pattern that passes the smoke gate (matches its
|
|
1510
|
+
// own badExample) but does NOT match the lesson's Example Hit.
|
|
1511
|
+
// verifyRuleExamples fails; the retry path fires. Attempt 2 returns a
|
|
1512
|
+
// pattern that matches both the badExample and the Example Hit.
|
|
1513
|
+
const lessonWithExample = {
|
|
1514
|
+
index: 0,
|
|
1515
|
+
heading: 'No console.log in production',
|
|
1516
|
+
body: 'Do not use console.log in production.\n**Example Hit:** console.log(x)',
|
|
1517
|
+
hash: 'h-retry-verify',
|
|
1518
|
+
};
|
|
1519
|
+
const parseMock = vi
|
|
1520
|
+
.fn()
|
|
1521
|
+
.mockReturnValueOnce({
|
|
1522
|
+
compilable: true,
|
|
1523
|
+
pattern: 'zzz_only_matches_itself',
|
|
1524
|
+
message: 'No console.log',
|
|
1525
|
+
engine: 'regex',
|
|
1526
|
+
badExample: 'zzz_only_matches_itself',
|
|
1527
|
+
})
|
|
1528
|
+
.mockReturnValueOnce({
|
|
1529
|
+
compilable: true,
|
|
1530
|
+
pattern: 'console\\.log',
|
|
1531
|
+
message: 'No console.log',
|
|
1532
|
+
engine: 'regex',
|
|
1533
|
+
badExample: 'console.log("x")',
|
|
1534
|
+
});
|
|
1535
|
+
const orchestratorMock = vi
|
|
1536
|
+
.fn()
|
|
1537
|
+
.mockResolvedValueOnce('attempt-1')
|
|
1538
|
+
.mockResolvedValueOnce('attempt-2');
|
|
1539
|
+
const deps = {
|
|
1540
|
+
parseCompilerResponse: parseMock,
|
|
1541
|
+
runOrchestrator: orchestratorMock,
|
|
1542
|
+
existingByHash: new Map(),
|
|
1543
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1544
|
+
};
|
|
1545
|
+
const result = await compileLesson(lessonWithExample, 'system prompt', deps);
|
|
1546
|
+
expect(result.status).toBe('compiled');
|
|
1547
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(2);
|
|
1548
|
+
// The retry directive must reflect the Example Hit failure (generic
|
|
1549
|
+
// wording so it can serve both smoke-gate and verify failures), it
|
|
1550
|
+
// must carry the prior pattern so the LLM sees what it produced, and
|
|
1551
|
+
// — when the failure source is verifyRuleExamples — the "Code
|
|
1552
|
+
// snippet the pattern had to match" field must show the missed
|
|
1553
|
+
// Example Hit line, not the LLM's own badExample (which the pattern
|
|
1554
|
+
// did match, since the smoke gate passed).
|
|
1555
|
+
const userPrompt2 = orchestratorMock.mock.calls[1][0];
|
|
1556
|
+
expect(userPrompt2).toContain('Previous Attempt Failed Verification');
|
|
1557
|
+
expect(userPrompt2).toContain('zzz_only_matches_itself'); // pattern field
|
|
1558
|
+
expect(userPrompt2).toContain('console.log(x)'); // missed Example Hit
|
|
1559
|
+
});
|
|
1560
|
+
it('exhausts retries when Example Hit/Miss verification fails on every attempt', async () => {
|
|
1561
|
+
// Same Example Hit lesson as above, but the LLM never produces a
|
|
1562
|
+
// pattern that matches the lesson's Example Hit. Three attempts, each
|
|
1563
|
+
// passing the smoke gate but failing verifyRuleExamples. Expected
|
|
1564
|
+
// outcome: skipped, reasonCode 'verify-retry-exhausted'.
|
|
1565
|
+
const lessonWithExample = {
|
|
1566
|
+
index: 0,
|
|
1567
|
+
heading: 'No console.log in production',
|
|
1568
|
+
body: 'Do not use console.log in production.\n**Example Hit:** console.log(x)',
|
|
1569
|
+
hash: 'h-verify-exhaust',
|
|
1570
|
+
};
|
|
1571
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1572
|
+
compilable: true,
|
|
1573
|
+
pattern: 'zzz_only_matches_itself',
|
|
1574
|
+
message: 'No console.log',
|
|
1575
|
+
engine: 'regex',
|
|
1576
|
+
badExample: 'zzz_only_matches_itself',
|
|
1577
|
+
});
|
|
1578
|
+
const orchestratorMock = vi.fn().mockResolvedValue('always-misses');
|
|
1579
|
+
const deps = {
|
|
1580
|
+
parseCompilerResponse: parseMock,
|
|
1581
|
+
runOrchestrator: orchestratorMock,
|
|
1582
|
+
existingByHash: new Map(),
|
|
1583
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1584
|
+
};
|
|
1585
|
+
const result = await compileLesson(lessonWithExample, 'system prompt', deps);
|
|
1586
|
+
expect(result.status).toBe('skipped');
|
|
1587
|
+
if (result.status === 'skipped') {
|
|
1588
|
+
expect(result.reasonCode).toBe('verify-retry-exhausted');
|
|
1589
|
+
}
|
|
1590
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(3);
|
|
1591
|
+
});
|
|
1592
|
+
it('returns skipped with pattern-syntax-invalid when the LLM emits an invalid regex (validator rejection)', async () => {
|
|
1593
|
+
// Validator-level rejection (ADR-088 Layer 4, mmnto-ai/totem#1481).
|
|
1594
|
+
// Retrying an invalid regex produces more invalid regexes and wastes
|
|
1595
|
+
// tokens, so no retry fires. Pre-#1481 this returned 'failed'; post-
|
|
1596
|
+
// #1481 it returns 'skipped' with reasonCode 'pattern-syntax-invalid'
|
|
1597
|
+
// so the ledger carries a machine-readable record per the Layer 4
|
|
1598
|
+
// explicit-failure contract.
|
|
1599
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1600
|
+
compilable: true,
|
|
1601
|
+
pattern: '[unclosed-bracket-class',
|
|
1602
|
+
message: 'Invalid regex test',
|
|
1603
|
+
engine: 'regex',
|
|
1604
|
+
badExample: 'any string',
|
|
1605
|
+
});
|
|
1606
|
+
const orchestratorMock = vi.fn().mockResolvedValue('invalid-regex-output');
|
|
1607
|
+
const deps = {
|
|
1608
|
+
parseCompilerResponse: parseMock,
|
|
1609
|
+
runOrchestrator: orchestratorMock,
|
|
1610
|
+
existingByHash: new Map(),
|
|
1611
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1612
|
+
};
|
|
1613
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1614
|
+
expect(result.status).toBe('skipped');
|
|
1615
|
+
if (result.status === 'skipped') {
|
|
1616
|
+
expect(result.reasonCode).toBe('pattern-syntax-invalid');
|
|
1617
|
+
expect(result.reason).toContain('Rejected regex');
|
|
1618
|
+
}
|
|
1619
|
+
expect(orchestratorMock).toHaveBeenCalledTimes(1);
|
|
1620
|
+
});
|
|
1621
|
+
});
|
|
1622
|
+
// ─── ADR-088 Phase 1 Layer 3: unverified flag (mmnto-ai/totem#1480) ──
|
|
1623
|
+
describe('compileLesson unverified flag', () => {
|
|
1624
|
+
const lessonWithoutExampleHit = {
|
|
1625
|
+
index: 0,
|
|
1626
|
+
heading: 'Use err not error in catch blocks',
|
|
1627
|
+
body: 'Always use `err` as the variable name in catch blocks.',
|
|
1628
|
+
hash: 'h-no-example',
|
|
1629
|
+
};
|
|
1630
|
+
const lessonWithExampleHit = {
|
|
1631
|
+
index: 0,
|
|
1632
|
+
heading: 'No console.log in production',
|
|
1633
|
+
body: 'Do not ship console.log.\n**Example Hit:** console.log(x)',
|
|
1634
|
+
hash: 'h-with-example',
|
|
1635
|
+
};
|
|
1636
|
+
it('flags non-security Pipeline 2 rules as unverified when the lesson has no Example Hit', async () => {
|
|
1637
|
+
// Invariant #2: non-security rule without Example Hit ships with
|
|
1638
|
+
// `unverified: true`. Severity passes through from the LLM output —
|
|
1639
|
+
// no warning-downgrade happens here. A doctor advisory for
|
|
1640
|
+
// `unverified: true` + `severity: 'error'` combos ships with #1483.
|
|
1641
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1642
|
+
compilable: true,
|
|
1643
|
+
pattern: 'console\\.log',
|
|
1644
|
+
message: 'No console.log',
|
|
1645
|
+
engine: 'regex',
|
|
1646
|
+
severity: 'error',
|
|
1647
|
+
badExample: 'console.log(x)',
|
|
1648
|
+
});
|
|
1649
|
+
const orchestratorMock = vi.fn().mockResolvedValue('{"compilable": true}');
|
|
1650
|
+
const deps = {
|
|
1651
|
+
parseCompilerResponse: parseMock,
|
|
1652
|
+
runOrchestrator: orchestratorMock,
|
|
1653
|
+
existingByHash: new Map(),
|
|
1654
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1655
|
+
};
|
|
1656
|
+
const result = await compileLesson(lessonWithoutExampleHit, 'system prompt', deps);
|
|
1657
|
+
expect(result.status).toBe('compiled');
|
|
1658
|
+
if (result.status === 'compiled') {
|
|
1659
|
+
expect(result.rule.unverified).toBe(true);
|
|
1660
|
+
expect(result.rule.severity).toBe('error');
|
|
1661
|
+
}
|
|
1662
|
+
});
|
|
1663
|
+
it('applies the general omitted-severity default on an unverified rule', async () => {
|
|
1664
|
+
// Companion to the severity-pass-through invariant. When the LLM emits
|
|
1665
|
+
// no severity, buildCompiledRule applies the general `'warning'`
|
|
1666
|
+
// default (compile-lesson.ts line ~299: `parsed.severity ?? 'warning'`).
|
|
1667
|
+
// That default is not ADR-088 specific: it fires for any rule without
|
|
1668
|
+
// an emitted severity, unverified or not. Pinning it here guards
|
|
1669
|
+
// against a future ADR-088 change that tries to force severity in
|
|
1670
|
+
// addition to the existing default.
|
|
1671
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1672
|
+
compilable: true,
|
|
1673
|
+
pattern: 'console\\.log',
|
|
1674
|
+
message: 'No console.log',
|
|
1675
|
+
engine: 'regex',
|
|
1676
|
+
badExample: 'console.log(x)',
|
|
1677
|
+
});
|
|
1678
|
+
const orchestratorMock = vi.fn().mockResolvedValue('{"compilable": true}');
|
|
1679
|
+
const deps = {
|
|
1680
|
+
parseCompilerResponse: parseMock,
|
|
1681
|
+
runOrchestrator: orchestratorMock,
|
|
1682
|
+
existingByHash: new Map(),
|
|
1683
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1684
|
+
};
|
|
1685
|
+
const result = await compileLesson(lessonWithoutExampleHit, 'system prompt', deps);
|
|
1686
|
+
expect(result.status).toBe('compiled');
|
|
1687
|
+
if (result.status === 'compiled') {
|
|
1688
|
+
expect(result.rule.unverified).toBe(true);
|
|
1689
|
+
expect(result.rule.severity).toBe('warning');
|
|
1690
|
+
}
|
|
1691
|
+
});
|
|
1692
|
+
it('leaves unverified absent when the lesson carries a non-empty Example Hit', async () => {
|
|
1693
|
+
// Invariant #1: presence of Example Hit produces unverified: undefined.
|
|
1694
|
+
// The serialized JSON omits the field — `canonicalStringify`
|
|
1695
|
+
// (key-sorted JSON.stringify) writes nothing for undefined values so
|
|
1696
|
+
// pre-#1480 manifest hashes stay stable.
|
|
1697
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1698
|
+
compilable: true,
|
|
1699
|
+
pattern: 'console\\.log',
|
|
1700
|
+
message: 'No console.log',
|
|
1701
|
+
engine: 'regex',
|
|
1702
|
+
badExample: 'console.log(x)',
|
|
1703
|
+
});
|
|
1704
|
+
const orchestratorMock = vi.fn().mockResolvedValue('{"compilable": true}');
|
|
1705
|
+
const deps = {
|
|
1706
|
+
parseCompilerResponse: parseMock,
|
|
1707
|
+
runOrchestrator: orchestratorMock,
|
|
1708
|
+
existingByHash: new Map(),
|
|
1709
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1710
|
+
};
|
|
1711
|
+
const result = await compileLesson(lessonWithExampleHit, 'system prompt', deps);
|
|
1712
|
+
expect(result.status).toBe('compiled');
|
|
1713
|
+
if (result.status === 'compiled') {
|
|
1714
|
+
expect(result.rule.unverified).toBeUndefined();
|
|
1715
|
+
}
|
|
1716
|
+
});
|
|
1717
|
+
it('treats an Example Hit that is whitespace-only as absent for the unverified flag', async () => {
|
|
1718
|
+
// Edge case from spec: `**Example Hit:** ` (whitespace-only value) is
|
|
1719
|
+
// treated as no ground truth for the `unverified` signal. `trim()` on
|
|
1720
|
+
// every extracted line before counting guards against false-negatives
|
|
1721
|
+
// on whitespace-only examples. (The existing verify step still runs
|
|
1722
|
+
// against whatever extractRuleExamples returns, so the test pattern
|
|
1723
|
+
// matches the empty string to avoid tangling the test with verify
|
|
1724
|
+
// semantics — scope is the unverified flag only.)
|
|
1725
|
+
const lessonEmptyExample = {
|
|
1726
|
+
index: 0,
|
|
1727
|
+
heading: 'Edge case lesson',
|
|
1728
|
+
body: 'Body text.\n**Example Hit:** ',
|
|
1729
|
+
hash: 'h-empty-example',
|
|
1730
|
+
};
|
|
1731
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1732
|
+
compilable: true,
|
|
1733
|
+
// Pattern that matches any string — including the trimmed-empty
|
|
1734
|
+
// Example Hit — so verifyRuleExamples passes and we can assert
|
|
1735
|
+
// on the unverified flag without the verify branch interfering.
|
|
1736
|
+
pattern: '.*',
|
|
1737
|
+
message: 'msg',
|
|
1738
|
+
engine: 'regex',
|
|
1739
|
+
badExample: 'anything',
|
|
1740
|
+
});
|
|
1741
|
+
const orchestratorMock = vi.fn().mockResolvedValue('{"compilable": true}');
|
|
1742
|
+
const deps = {
|
|
1743
|
+
parseCompilerResponse: parseMock,
|
|
1744
|
+
runOrchestrator: orchestratorMock,
|
|
1745
|
+
existingByHash: new Map(),
|
|
1746
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1747
|
+
};
|
|
1748
|
+
const result = await compileLesson(lessonEmptyExample, 'system prompt', deps);
|
|
1749
|
+
expect(result.status).toBe('compiled');
|
|
1750
|
+
if (result.status === 'compiled') {
|
|
1751
|
+
expect(result.rule.unverified).toBe(true);
|
|
1752
|
+
}
|
|
1753
|
+
});
|
|
1754
|
+
it('rejects security-context lessons that lack an Example Hit with security-rule-rejected', async () => {
|
|
1755
|
+
// Invariant #3: security-context lesson without Example Hit produces
|
|
1756
|
+
// no rule; ledger entry carries reasonCode 'security-rule-rejected'.
|
|
1757
|
+
// No orchestrator call fires (short-circuit before Pipeline 2).
|
|
1758
|
+
const orchestratorMock = vi.fn();
|
|
1759
|
+
const deps = {
|
|
1760
|
+
parseCompilerResponse: vi.fn(),
|
|
1761
|
+
runOrchestrator: orchestratorMock,
|
|
1762
|
+
existingByHash: new Map(),
|
|
1763
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1764
|
+
securityContext: true,
|
|
1765
|
+
};
|
|
1766
|
+
const result = await compileLesson(lessonWithoutExampleHit, 'system prompt', deps);
|
|
1767
|
+
expect(result.status).toBe('skipped');
|
|
1768
|
+
if (result.status === 'skipped') {
|
|
1769
|
+
expect(result.reasonCode).toBe('security-rule-rejected');
|
|
1770
|
+
expect(result.reason).toContain('Example Hit');
|
|
1771
|
+
}
|
|
1772
|
+
expect(orchestratorMock).not.toHaveBeenCalled();
|
|
1773
|
+
});
|
|
1774
|
+
it('flags Pipeline 1 manual rules as unverified when the lesson has no Example Hit', async () => {
|
|
1775
|
+
// Pipeline 1 consistency (design doc Open Question 2, resolution (a)):
|
|
1776
|
+
// manual rules authored without an Example Hit ship unverified too.
|
|
1777
|
+
// The #1414 backfill sweep will eliminate this population eventually.
|
|
1778
|
+
const manualLessonNoExample = {
|
|
1779
|
+
index: 0,
|
|
1780
|
+
heading: 'No console.log in production',
|
|
1781
|
+
body: '**Pattern:** `console\\.log`\n**Engine:** regex\n**Severity:** warning\n**Scope:** **/*.ts',
|
|
1782
|
+
hash: 'h-manual-no-example',
|
|
1783
|
+
};
|
|
1784
|
+
const deps = {
|
|
1785
|
+
parseCompilerResponse: vi.fn(),
|
|
1786
|
+
runOrchestrator: vi.fn(),
|
|
1787
|
+
existingByHash: new Map(),
|
|
1788
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1789
|
+
};
|
|
1790
|
+
const result = await compileLesson(manualLessonNoExample, 'system prompt', deps);
|
|
1791
|
+
expect(result.status).toBe('compiled');
|
|
1792
|
+
if (result.status === 'compiled') {
|
|
1793
|
+
expect(result.rule.unverified).toBe(true);
|
|
1794
|
+
}
|
|
1795
|
+
expect(deps.runOrchestrator).not.toHaveBeenCalled();
|
|
1796
|
+
});
|
|
1797
|
+
});
|
|
1798
|
+
// ─── ADR-088 Decision 3: security-context signal (mmnto-ai/totem#1480) ──
|
|
1799
|
+
describe('isSecurityContext', () => {
|
|
1800
|
+
// Covers both signals the helper accepts. The `deps.securityContext`
|
|
1801
|
+
// branch is exercised end-to-end by compileLesson tests above; the
|
|
1802
|
+
// `rule.immutable === true` branch is defense-in-depth for pack-merged
|
|
1803
|
+
// rules (ADR-089) and cannot reach compileLesson's public surface today
|
|
1804
|
+
// because CompilerOutput has no immutable field and the manual pattern
|
|
1805
|
+
// parser does not parse `**Immutable:**`. Unit tests on the helper lock
|
|
1806
|
+
// both signals so a future CompilerOutput or parser change cannot drop
|
|
1807
|
+
// the rule-based branch silently.
|
|
1808
|
+
const baseDeps = {
|
|
1809
|
+
parseCompilerResponse: vi.fn(),
|
|
1810
|
+
runOrchestrator: vi.fn(),
|
|
1811
|
+
existingByHash: new Map(),
|
|
1812
|
+
};
|
|
1813
|
+
it('returns true when deps.securityContext is true', () => {
|
|
1814
|
+
expect(isSecurityContext({ ...baseDeps, securityContext: true }, null)).toBe(true);
|
|
1815
|
+
});
|
|
1816
|
+
it('returns true when the built rule carries immutable: true', () => {
|
|
1817
|
+
expect(isSecurityContext(baseDeps, { immutable: true })).toBe(true);
|
|
1818
|
+
});
|
|
1819
|
+
it('returns true when both signals are present', () => {
|
|
1820
|
+
expect(isSecurityContext({ ...baseDeps, securityContext: true }, { immutable: true })).toBe(true);
|
|
1821
|
+
});
|
|
1822
|
+
it('returns false when neither signal is present', () => {
|
|
1823
|
+
expect(isSecurityContext(baseDeps, null)).toBe(false);
|
|
1824
|
+
expect(isSecurityContext(baseDeps, { immutable: false })).toBe(false);
|
|
1825
|
+
expect(isSecurityContext(baseDeps, {})).toBe(false);
|
|
1826
|
+
});
|
|
1827
|
+
});
|
|
1828
|
+
// ─── Layer-trace events (mmnto-ai/totem#1482) ────────────
|
|
1829
|
+
describe('compileLesson trace events', () => {
|
|
1830
|
+
it('Pipeline 1 manual compile emits a single layer-1 result event', async () => {
|
|
1831
|
+
// Invariant: Pipeline 1 is deterministic (no LLM, no retry). One event,
|
|
1832
|
+
// outcome 'compiled', layer 1.
|
|
1833
|
+
const deps = {
|
|
1834
|
+
parseCompilerResponse: vi.fn(),
|
|
1835
|
+
runOrchestrator: vi.fn(),
|
|
1836
|
+
existingByHash: new Map(),
|
|
1837
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1838
|
+
};
|
|
1839
|
+
const result = await compileLesson(manualLesson, 'system prompt', deps);
|
|
1840
|
+
expect(result.status).toBe('compiled');
|
|
1841
|
+
expect(result.trace).toBeDefined();
|
|
1842
|
+
expect(result.trace).toHaveLength(1);
|
|
1843
|
+
expect(result.trace[0]).toMatchObject({
|
|
1844
|
+
layer: 1,
|
|
1845
|
+
action: 'result',
|
|
1846
|
+
outcome: 'compiled',
|
|
1847
|
+
});
|
|
1848
|
+
expect(deps.runOrchestrator).not.toHaveBeenCalled();
|
|
1849
|
+
});
|
|
1850
|
+
it('Pipeline 2 first-try compile emits generate + verify(MATCH) + result(compiled)', async () => {
|
|
1851
|
+
const lessonOk = {
|
|
1852
|
+
index: 0,
|
|
1853
|
+
heading: 'No console.log in production',
|
|
1854
|
+
body: 'Do not use console.log.\n**Example Hit:** console.log(x)',
|
|
1855
|
+
hash: 'h-trace-first-try',
|
|
1856
|
+
};
|
|
1857
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1858
|
+
compilable: true,
|
|
1859
|
+
pattern: 'console\\.log',
|
|
1860
|
+
message: 'No console.log',
|
|
1861
|
+
engine: 'regex',
|
|
1862
|
+
badExample: 'console.log("x")',
|
|
1863
|
+
});
|
|
1864
|
+
const orchestratorMock = vi.fn().mockResolvedValue('attempt-1');
|
|
1865
|
+
const deps = {
|
|
1866
|
+
parseCompilerResponse: parseMock,
|
|
1867
|
+
runOrchestrator: orchestratorMock,
|
|
1868
|
+
existingByHash: new Map(),
|
|
1869
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1870
|
+
};
|
|
1871
|
+
const result = await compileLesson(lessonOk, 'system prompt', deps);
|
|
1872
|
+
expect(result.status).toBe('compiled');
|
|
1873
|
+
expect(result.trace).toBeDefined();
|
|
1874
|
+
expect(result.trace).toHaveLength(3);
|
|
1875
|
+
const [gen, verify, res] = result.trace;
|
|
1876
|
+
expect(gen).toMatchObject({ layer: 3, action: 'generate', outcome: 'attempt-1' });
|
|
1877
|
+
expect(gen.patternHash).toMatch(/^[0-9a-f]{16}$/);
|
|
1878
|
+
expect(verify).toMatchObject({ layer: 3, action: 'verify', outcome: 'MATCH' });
|
|
1879
|
+
expect(res).toMatchObject({ layer: 3, action: 'result', outcome: 'compiled' });
|
|
1880
|
+
});
|
|
1881
|
+
it('Pipeline 2 verify-retry-exhausted emits generate+verify per attempt plus retry scheduling and terminal result', async () => {
|
|
1882
|
+
// Three attempts, each passing smoke gate but failing verifyRuleExamples.
|
|
1883
|
+
// Trace should contain (generate, verify, retry) for attempts 1 and 2, then
|
|
1884
|
+
// (generate, verify, result) for attempt 3.
|
|
1885
|
+
const lessonWithExample = {
|
|
1886
|
+
index: 0,
|
|
1887
|
+
heading: 'No console.log in production',
|
|
1888
|
+
body: 'Do not use console.log in production.\n**Example Hit:** console.log(x)',
|
|
1889
|
+
hash: 'h-trace-exhaust',
|
|
1890
|
+
};
|
|
1891
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1892
|
+
compilable: true,
|
|
1893
|
+
pattern: 'zzz_only_matches_itself',
|
|
1894
|
+
message: 'No console.log',
|
|
1895
|
+
engine: 'regex',
|
|
1896
|
+
badExample: 'zzz_only_matches_itself',
|
|
1897
|
+
});
|
|
1898
|
+
const orchestratorMock = vi.fn().mockResolvedValue('always-misses');
|
|
1899
|
+
const deps = {
|
|
1900
|
+
parseCompilerResponse: parseMock,
|
|
1901
|
+
runOrchestrator: orchestratorMock,
|
|
1902
|
+
existingByHash: new Map(),
|
|
1903
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1904
|
+
};
|
|
1905
|
+
const result = await compileLesson(lessonWithExample, 'system prompt', deps);
|
|
1906
|
+
expect(result.status).toBe('skipped');
|
|
1907
|
+
expect(result.trace).toBeDefined();
|
|
1908
|
+
// 3 attempts × (generate + verify) = 6 events, plus 2 retry events between
|
|
1909
|
+
// the first two attempts, plus 1 terminal result = 9 events total.
|
|
1910
|
+
const actions = result.trace.map((e) => e.action);
|
|
1911
|
+
expect(actions).toEqual([
|
|
1912
|
+
'generate',
|
|
1913
|
+
'verify',
|
|
1914
|
+
'retry',
|
|
1915
|
+
'generate',
|
|
1916
|
+
'verify',
|
|
1917
|
+
'retry',
|
|
1918
|
+
'generate',
|
|
1919
|
+
'verify',
|
|
1920
|
+
'result',
|
|
1921
|
+
]);
|
|
1922
|
+
const terminal = result.trace[result.trace.length - 1];
|
|
1923
|
+
expect(terminal).toMatchObject({
|
|
1924
|
+
layer: 3,
|
|
1925
|
+
action: 'result',
|
|
1926
|
+
outcome: 'skipped',
|
|
1927
|
+
reasonCode: 'verify-retry-exhausted',
|
|
1928
|
+
});
|
|
1929
|
+
});
|
|
1930
|
+
it('Pipeline 2 out-of-scope skip emits a single layer-3 result event with reasonCode', async () => {
|
|
1931
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1932
|
+
compilable: false,
|
|
1933
|
+
reason: 'Architectural principle, not a pattern.',
|
|
1934
|
+
});
|
|
1935
|
+
const orchestratorMock = vi.fn().mockResolvedValue('{"compilable": false}');
|
|
1936
|
+
const deps = {
|
|
1937
|
+
parseCompilerResponse: parseMock,
|
|
1938
|
+
runOrchestrator: orchestratorMock,
|
|
1939
|
+
existingByHash: new Map(),
|
|
1940
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1941
|
+
};
|
|
1942
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1943
|
+
expect(result.status).toBe('skipped');
|
|
1944
|
+
expect(result.trace).toHaveLength(1);
|
|
1945
|
+
expect(result.trace[0]).toMatchObject({
|
|
1946
|
+
layer: 3,
|
|
1947
|
+
action: 'result',
|
|
1948
|
+
outcome: 'skipped',
|
|
1949
|
+
reasonCode: 'out-of-scope',
|
|
1950
|
+
});
|
|
1951
|
+
});
|
|
1952
|
+
it('Pipeline 2 validator rejection emits generate + verify(validator-rejected) + result(skipped)', async () => {
|
|
1953
|
+
// Invalid regex is terminal. No retry fires. Trace should show the
|
|
1954
|
+
// generate event then the validator-rejected verify event then result.
|
|
1955
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
1956
|
+
compilable: true,
|
|
1957
|
+
pattern: '[unclosed-bracket',
|
|
1958
|
+
message: 'bad regex',
|
|
1959
|
+
engine: 'regex',
|
|
1960
|
+
badExample: 'anything',
|
|
1961
|
+
});
|
|
1962
|
+
const orchestratorMock = vi.fn().mockResolvedValue('invalid-regex-output');
|
|
1963
|
+
const deps = {
|
|
1964
|
+
parseCompilerResponse: parseMock,
|
|
1965
|
+
runOrchestrator: orchestratorMock,
|
|
1966
|
+
existingByHash: new Map(),
|
|
1967
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
1968
|
+
};
|
|
1969
|
+
const result = await compileLesson(lesson, 'system prompt', deps);
|
|
1970
|
+
expect(result.status).toBe('skipped');
|
|
1971
|
+
const actions = result.trace.map((e) => e.action);
|
|
1972
|
+
expect(actions).toEqual(['generate', 'verify', 'result']);
|
|
1973
|
+
expect(result.trace[1]).toMatchObject({
|
|
1974
|
+
layer: 3,
|
|
1975
|
+
action: 'verify',
|
|
1976
|
+
outcome: 'validator-rejected',
|
|
1977
|
+
});
|
|
1978
|
+
expect(result.trace[2]).toMatchObject({
|
|
1979
|
+
layer: 3,
|
|
1980
|
+
action: 'result',
|
|
1981
|
+
outcome: 'skipped',
|
|
1982
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
1983
|
+
});
|
|
1984
|
+
});
|
|
1985
|
+
it('Pipeline 3 example-based compile emits generate + verify + result events at layer 2', async () => {
|
|
1986
|
+
const pipeline3Lesson = {
|
|
1987
|
+
index: 0,
|
|
1988
|
+
heading: 'Example-based lesson',
|
|
1989
|
+
body: [
|
|
1990
|
+
'Do not use console.log.',
|
|
1991
|
+
'',
|
|
1992
|
+
'**Bad:**',
|
|
1993
|
+
'```ts',
|
|
1994
|
+
'console.log("x")',
|
|
1995
|
+
'```',
|
|
1996
|
+
'',
|
|
1997
|
+
'**Good:**',
|
|
1998
|
+
'```ts',
|
|
1999
|
+
'log.info("x")',
|
|
2000
|
+
'```',
|
|
2001
|
+
].join('\n'),
|
|
2002
|
+
hash: 'h-trace-pipeline3',
|
|
2003
|
+
};
|
|
2004
|
+
const parseMock = vi.fn().mockReturnValue({
|
|
2005
|
+
compilable: true,
|
|
2006
|
+
pattern: 'console\\.log',
|
|
2007
|
+
message: 'No console.log',
|
|
2008
|
+
engine: 'regex',
|
|
2009
|
+
badExample: 'console.log("x")',
|
|
2010
|
+
});
|
|
2011
|
+
const orchestratorMock = vi.fn().mockResolvedValue('pipeline-3-response');
|
|
2012
|
+
const deps = {
|
|
2013
|
+
parseCompilerResponse: parseMock,
|
|
2014
|
+
runOrchestrator: orchestratorMock,
|
|
2015
|
+
existingByHash: new Map(),
|
|
2016
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
2017
|
+
};
|
|
2018
|
+
const result = await compileLesson(pipeline3Lesson, 'system prompt', deps);
|
|
2019
|
+
expect(result.status).toBe('compiled');
|
|
2020
|
+
const actions = result.trace.map((e) => e.action);
|
|
2021
|
+
expect(actions).toEqual(['generate', 'verify', 'result']);
|
|
2022
|
+
// Pipeline 3 emits at layer 2 per the design doc reservation.
|
|
2023
|
+
expect(result.trace[0].layer).toBe(2);
|
|
2024
|
+
expect(result.trace[0].patternHash).toMatch(/^[0-9a-f]{16}$/);
|
|
2025
|
+
});
|
|
2026
|
+
it('security-context short-circuit emits a single result event with security-rule-rejected', async () => {
|
|
2027
|
+
const securityLesson = {
|
|
2028
|
+
index: 0,
|
|
2029
|
+
heading: 'No shell injection',
|
|
2030
|
+
body: 'Never pass user input to spawn.',
|
|
2031
|
+
hash: 'h-trace-security-short-circuit',
|
|
2032
|
+
};
|
|
2033
|
+
const orchestratorMock = vi.fn();
|
|
2034
|
+
const deps = {
|
|
2035
|
+
parseCompilerResponse: vi.fn(),
|
|
2036
|
+
runOrchestrator: orchestratorMock,
|
|
2037
|
+
existingByHash: new Map(),
|
|
2038
|
+
callbacks: { onWarn: vi.fn(), onDim: vi.fn() },
|
|
2039
|
+
securityContext: true,
|
|
2040
|
+
};
|
|
2041
|
+
const result = await compileLesson(securityLesson, 'system prompt', deps);
|
|
2042
|
+
expect(result.status).toBe('skipped');
|
|
2043
|
+
expect(result.trace).toHaveLength(1);
|
|
2044
|
+
expect(result.trace[0]).toMatchObject({
|
|
2045
|
+
layer: 3,
|
|
2046
|
+
action: 'result',
|
|
2047
|
+
outcome: 'skipped',
|
|
2048
|
+
reasonCode: 'security-rule-rejected',
|
|
2049
|
+
});
|
|
2050
|
+
expect(orchestratorMock).not.toHaveBeenCalled();
|
|
2051
|
+
});
|
|
2052
|
+
});
|
|
1305
2053
|
//# sourceMappingURL=compile-lesson.test.js.map
|