@mmnto/totem 1.14.10 → 1.14.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compile-lesson.d.ts +76 -1
- package/dist/compile-lesson.d.ts.map +1 -1
- package/dist/compile-lesson.js +481 -53
- package/dist/compile-lesson.js.map +1 -1
- package/dist/compile-lesson.test.js +756 -8
- package/dist/compile-lesson.test.js.map +1 -1
- package/dist/compiler-schema.d.ts +185 -9
- package/dist/compiler-schema.d.ts.map +1 -1
- package/dist/compiler-schema.js +95 -10
- package/dist/compiler-schema.js.map +1 -1
- package/dist/compiler.d.ts +11 -3
- package/dist/compiler.d.ts.map +1 -1
- package/dist/compiler.js +24 -4
- package/dist/compiler.js.map +1 -1
- package/dist/compiler.test.js +162 -22
- package/dist/compiler.test.js.map +1 -1
- package/dist/config-schema.d.ts +86 -0
- package/dist/config-schema.d.ts.map +1 -1
- package/dist/config-schema.js +54 -0
- package/dist/config-schema.js.map +1 -1
- package/dist/config-schema.test.js +137 -1
- package/dist/config-schema.test.js.map +1 -1
- package/dist/index.d.ts +8 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/ledger.d.ts +10 -0
- package/dist/ledger.d.ts.map +1 -1
- package/dist/ledger.js +8 -0
- package/dist/ledger.js.map +1 -1
- package/dist/lesson-pattern.d.ts.map +1 -1
- package/dist/lesson-pattern.js +6 -9
- package/dist/lesson-pattern.js.map +1 -1
- package/dist/pack-merge.d.ts +73 -0
- package/dist/pack-merge.d.ts.map +1 -0
- package/dist/pack-merge.js +117 -0
- package/dist/pack-merge.js.map +1 -0
- package/dist/pack-merge.test.d.ts +2 -0
- package/dist/pack-merge.test.d.ts.map +1 -0
- package/dist/pack-merge.test.js +238 -0
- package/dist/pack-merge.test.js.map +1 -0
- package/dist/regex-utils.d.ts +5 -0
- package/dist/regex-utils.d.ts.map +1 -1
- package/dist/regex-utils.js +6 -1
- package/dist/regex-utils.js.map +1 -1
- package/dist/regex-utils.test.js +15 -2
- package/dist/regex-utils.test.js.map +1 -1
- package/dist/rule-engine.d.ts.map +1 -1
- package/dist/rule-engine.js +3 -0
- package/dist/rule-engine.js.map +1 -1
- package/dist/rule-engine.test.js +29 -0
- package/dist/rule-engine.test.js.map +1 -1
- package/dist/rule-metrics.d.ts +40 -0
- package/dist/rule-metrics.d.ts.map +1 -1
- package/dist/rule-metrics.js +28 -0
- package/dist/rule-metrics.js.map +1 -1
- package/dist/rule-metrics.test.js +104 -3
- package/dist/rule-metrics.test.js.map +1 -1
- package/package.json +1 -1
package/dist/compile-lesson.js
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { Lang, parse } from '@ast-grep/napi';
|
|
2
3
|
import { runSmokeGate } from './compile-smoke-gate.js';
|
|
3
4
|
import { engineFields, sanitizeFileGlobs, validateRegex } from './compiler.js';
|
|
4
5
|
import { extractBadGoodSnippets, extractManualPattern, extractRuleExamples, } from './lesson-pattern.js';
|
|
5
6
|
import { testRule } from './rule-tester.js';
|
|
7
|
+
/** Produce a stable short hash of a pattern string for trace correlation. */
|
|
8
|
+
function hashPattern(pattern) {
|
|
9
|
+
return createHash('sha256').update(pattern).digest('hex').slice(0, 16);
|
|
10
|
+
}
|
|
6
11
|
// ─── ast-grep pattern validation ───────────────────
|
|
7
12
|
/**
|
|
8
13
|
* Compile-time validation for ast-grep patterns (#1062, #1339).
|
|
@@ -410,6 +415,40 @@ export function buildManualRule(lesson, existingByHash) {
|
|
|
410
415
|
};
|
|
411
416
|
}
|
|
412
417
|
// ─── Single-lesson compilation ──────────────────────
|
|
418
|
+
/**
|
|
419
|
+
* Check whether the lesson body carries a non-empty Example Hit block.
|
|
420
|
+
* ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): rules compiled without an
|
|
421
|
+
* Example Hit ship as `unverified: true` (non-security) or fail outright
|
|
422
|
+
* (security). Empty code fences like ```ts\n``` are treated as absent —
|
|
423
|
+
* `trim()` is applied before the length check.
|
|
424
|
+
*/
|
|
425
|
+
function hasExampleHits(body) {
|
|
426
|
+
const examples = extractRuleExamples(body);
|
|
427
|
+
if (!examples)
|
|
428
|
+
return false;
|
|
429
|
+
return examples.hits.some((line) => line.trim().length > 0);
|
|
430
|
+
}
|
|
431
|
+
/**
|
|
432
|
+
* Security-context signal for the missing-Example-Hit check. Either the
|
|
433
|
+
* compile orchestrator asserts the pack is security-scoped
|
|
434
|
+
* (`deps.securityContext === true`) OR the rule under construction already
|
|
435
|
+
* carries `immutable: true` (set by the pack manifest, ADR-089). Both
|
|
436
|
+
* paths trigger the zero-tolerance reject per ADR-088 Decision 3.
|
|
437
|
+
*
|
|
438
|
+
* The LLM-emitted `CompilerOutput` does not currently carry an `immutable`
|
|
439
|
+
* field (packs set it at pack-merge time), so the second signal only
|
|
440
|
+
* engages on the Pipeline 1 manual-rule path where `buildManualRule`
|
|
441
|
+
* could synthesize an immutable rule directly. A future change that
|
|
442
|
+
* threads `immutable` through `CompilerOutput` can wire Pipeline 2/3
|
|
443
|
+
* into this helper without touching the call sites.
|
|
444
|
+
*/
|
|
445
|
+
export function isSecurityContext(deps, rule) {
|
|
446
|
+
if (deps.securityContext === true)
|
|
447
|
+
return true;
|
|
448
|
+
if (rule?.immutable === true)
|
|
449
|
+
return true;
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
413
452
|
/**
|
|
414
453
|
* Compile a single lesson into a rule.
|
|
415
454
|
* Handles both manual patterns (zero LLM) and LLM-compiled patterns.
|
|
@@ -417,21 +456,79 @@ export function buildManualRule(lesson, existingByHash) {
|
|
|
417
456
|
*/
|
|
418
457
|
export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
419
458
|
const { parseCompilerResponse, runOrchestrator, existingByHash, callbacks } = deps;
|
|
459
|
+
const exampleHitPresent = hasExampleHits(lesson.body);
|
|
460
|
+
// Trace buffer (mmnto-ai/totem#1482). Populated unconditionally across all
|
|
461
|
+
// three pipelines so the CLI can choose to render it under --verbose
|
|
462
|
+
// without a per-call gate. Cost of empty-or-small arrays per lesson is
|
|
463
|
+
// negligible; gating would fork the test surface.
|
|
464
|
+
const trace = [];
|
|
420
465
|
// ── Pipeline 1: Manual pattern (zero LLM) ────────
|
|
421
466
|
const manualResult = buildManualRule(lesson, existingByHash);
|
|
422
467
|
if (manualResult.rule) {
|
|
468
|
+
// ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): security-scoped manual
|
|
469
|
+
// rules without an Example Hit are rejected outright.
|
|
470
|
+
if (!exampleHitPresent && isSecurityContext(deps, manualResult.rule)) {
|
|
471
|
+
const reason = 'Security rule missing Example Hit block (manual)';
|
|
472
|
+
callbacks?.onWarn?.(lesson.heading, `${reason} — rejecting`);
|
|
473
|
+
trace.push({
|
|
474
|
+
layer: 1,
|
|
475
|
+
action: 'result',
|
|
476
|
+
outcome: 'skipped',
|
|
477
|
+
reasonCode: 'security-rule-rejected',
|
|
478
|
+
});
|
|
479
|
+
return {
|
|
480
|
+
status: 'skipped',
|
|
481
|
+
hash: lesson.hash,
|
|
482
|
+
reason,
|
|
483
|
+
reasonCode: 'security-rule-rejected',
|
|
484
|
+
trace,
|
|
485
|
+
};
|
|
486
|
+
}
|
|
423
487
|
const testResult = verifyRuleExamples(manualResult.rule, lesson.body);
|
|
424
488
|
if (testResult && !testResult.passed) {
|
|
489
|
+
// A manual rule that failed its own inline examples is authoring
|
|
490
|
+
// error territory. Keep the existing 'failed' contract so the
|
|
491
|
+
// lesson stays pending rather than landing in nonCompilable.
|
|
492
|
+
// Pre-#1480 `verifyRuleExamples` fired on any hit or miss; the
|
|
493
|
+
// `hasExampleHits` check guards the unverified flag below without
|
|
494
|
+
// changing the verify gate itself.
|
|
425
495
|
callbacks?.onWarn?.(lesson.heading, formatExampleFailure(testResult));
|
|
426
|
-
|
|
496
|
+
trace.push({ layer: 1, action: 'result', outcome: 'failed' });
|
|
497
|
+
return { status: 'failed', trace };
|
|
498
|
+
}
|
|
499
|
+
if (!exampleHitPresent) {
|
|
500
|
+
manualResult.rule.unverified = true;
|
|
427
501
|
}
|
|
428
|
-
|
|
502
|
+
trace.push({ layer: 1, action: 'result', outcome: 'compiled' });
|
|
503
|
+
return { status: 'compiled', rule: manualResult.rule, trace };
|
|
429
504
|
}
|
|
430
505
|
if (manualResult.rejectReason) {
|
|
431
506
|
callbacks?.onWarn?.(lesson.heading, manualResult.rejectReason);
|
|
432
|
-
|
|
507
|
+
trace.push({ layer: 1, action: 'result', outcome: 'failed' });
|
|
508
|
+
return { status: 'failed', trace };
|
|
433
509
|
}
|
|
434
510
|
// manualResult.rule === null && no rejectReason → no manual pattern, proceed to Pipeline 3 or 2
|
|
511
|
+
// ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): security rules that enter
|
|
512
|
+
// the LLM path with no Example Hit short-circuit before the orchestrator
|
|
513
|
+
// call. Compile has no ground truth to verify against, and zero-tolerance
|
|
514
|
+
// per Decision 3 leaves nothing to retry.
|
|
515
|
+
if (!exampleHitPresent && deps.securityContext === true) {
|
|
516
|
+
const reason = 'Security rule missing Example Hit block';
|
|
517
|
+
callbacks?.onWarn?.(lesson.heading, `${reason} — rejecting`);
|
|
518
|
+
trace.push({
|
|
519
|
+
layer: 3,
|
|
520
|
+
action: 'result',
|
|
521
|
+
outcome: 'skipped',
|
|
522
|
+
reasonCode: 'security-rule-rejected',
|
|
523
|
+
});
|
|
524
|
+
return {
|
|
525
|
+
status: 'skipped',
|
|
526
|
+
hash: lesson.hash,
|
|
527
|
+
reason,
|
|
528
|
+
reasonCode: 'security-rule-rejected',
|
|
529
|
+
trace,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
435
532
|
// ── Pipeline 3: Example-based compilation (Bad/Good snippets) ──
|
|
436
533
|
const snippets = extractBadGoodSnippets(lesson.body);
|
|
437
534
|
if (snippets) {
|
|
@@ -455,17 +552,53 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
455
552
|
lesson.body,
|
|
456
553
|
].join('\n');
|
|
457
554
|
const response = await runOrchestrator(userPrompt, systemPrompt);
|
|
458
|
-
if (response == null)
|
|
459
|
-
|
|
555
|
+
if (response == null) {
|
|
556
|
+
trace.push({ layer: 2, action: 'result', outcome: 'noop' });
|
|
557
|
+
return { status: 'noop', trace };
|
|
558
|
+
}
|
|
460
559
|
const parsed = parseCompilerResponse(response);
|
|
461
560
|
if (!parsed) {
|
|
462
561
|
callbacks?.onWarn?.(lesson.heading, 'Pipeline 3: failed to parse LLM response — skipping');
|
|
463
|
-
|
|
562
|
+
trace.push({
|
|
563
|
+
layer: 2,
|
|
564
|
+
action: 'result',
|
|
565
|
+
outcome: 'skipped',
|
|
566
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
567
|
+
});
|
|
568
|
+
return {
|
|
569
|
+
status: 'skipped',
|
|
570
|
+
hash: lesson.hash,
|
|
571
|
+
reason: 'Pipeline 3: failed to parse LLM response',
|
|
572
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
573
|
+
trace,
|
|
574
|
+
};
|
|
464
575
|
}
|
|
465
576
|
if (!parsed.compilable) {
|
|
466
577
|
callbacks?.onDim?.(lesson.heading, 'Pipeline 3: not compilable — skipping');
|
|
467
|
-
|
|
578
|
+
trace.push({
|
|
579
|
+
layer: 2,
|
|
580
|
+
action: 'result',
|
|
581
|
+
outcome: 'skipped',
|
|
582
|
+
reasonCode: 'out-of-scope',
|
|
583
|
+
});
|
|
584
|
+
return {
|
|
585
|
+
status: 'skipped',
|
|
586
|
+
hash: lesson.hash,
|
|
587
|
+
reason: parsed.reason,
|
|
588
|
+
reasonCode: 'out-of-scope',
|
|
589
|
+
trace,
|
|
590
|
+
};
|
|
468
591
|
}
|
|
592
|
+
// Only include `patternHash` when the parsed output actually carries a
|
|
593
|
+
// pattern. Hashing the `(pattern unavailable)` fallback would make a
|
|
594
|
+
// failed generation look identical to a successful one in verbose output.
|
|
595
|
+
const pipeline3HasPattern = hasExtractablePattern(parsed);
|
|
596
|
+
trace.push({
|
|
597
|
+
layer: 2,
|
|
598
|
+
action: 'generate',
|
|
599
|
+
outcome: 'produced',
|
|
600
|
+
...(pipeline3HasPattern ? { patternHash: hashPattern(extractPatternString(parsed)) } : {}),
|
|
601
|
+
});
|
|
469
602
|
// mmnto/totem#1408: Pipeline 3 reuses its Bad snippet as the smoke-gate
|
|
470
603
|
// target. The LLM may or may not echo the snippet back in parsed.badExample;
|
|
471
604
|
// the override guarantees the gate has something to work with regardless.
|
|
@@ -474,8 +607,18 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
474
607
|
badExampleOverride: snippets.bad.join('\n'),
|
|
475
608
|
});
|
|
476
609
|
if (!ruleResult.rule) {
|
|
477
|
-
|
|
478
|
-
|
|
610
|
+
const rejectReason = ruleResult.rejectReason ?? 'Unknown error';
|
|
611
|
+
callbacks?.onWarn?.(lesson.heading, `Pipeline 3: ${rejectReason} — skipping`);
|
|
612
|
+
const reasonCode = classifyBuildRejectReason(rejectReason);
|
|
613
|
+
trace.push({ layer: 2, action: 'verify', outcome: 'rejected' });
|
|
614
|
+
trace.push({ layer: 2, action: 'result', outcome: 'skipped', reasonCode });
|
|
615
|
+
return {
|
|
616
|
+
status: 'skipped',
|
|
617
|
+
hash: lesson.hash,
|
|
618
|
+
reason: `Pipeline 3: ${rejectReason}`,
|
|
619
|
+
reasonCode,
|
|
620
|
+
trace,
|
|
621
|
+
};
|
|
479
622
|
}
|
|
480
623
|
// Self-verify: at least one Bad line should trigger, no Good line should trigger
|
|
481
624
|
const virtualPath = deriveVirtualFilePath(ruleResult.rule);
|
|
@@ -492,54 +635,339 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
492
635
|
const badCaught = snippets.bad.length - testResult.missedFails.length;
|
|
493
636
|
if (badCaught === 0 || testResult.falsePositives.length > 0) {
|
|
494
637
|
callbacks?.onWarn?.(lesson.heading, 'Pipeline 3: generated rule failed self-verification against Bad/Good snippets — skipping');
|
|
495
|
-
|
|
638
|
+
trace.push({ layer: 2, action: 'verify', outcome: 'self-test-failed' });
|
|
639
|
+
trace.push({ layer: 2, action: 'result', outcome: 'failed' });
|
|
640
|
+
return { status: 'failed', trace };
|
|
641
|
+
}
|
|
642
|
+
trace.push({ layer: 2, action: 'verify', outcome: 'passed' });
|
|
643
|
+
trace.push({ layer: 2, action: 'result', outcome: 'compiled' });
|
|
644
|
+
// Pipeline 3 rules are example-based — they always ship with a Bad snippet
|
|
645
|
+
// by construction. Flag unverified only when the lesson body itself omits
|
|
646
|
+
// the canonical Example Hit block, keeping the signal aligned with the
|
|
647
|
+
// lesson-level ground-truth rather than the Pipeline 3 self-test.
|
|
648
|
+
if (!exampleHitPresent) {
|
|
649
|
+
ruleResult.rule.unverified = true;
|
|
496
650
|
}
|
|
497
|
-
return { status: 'compiled', rule: ruleResult.rule };
|
|
651
|
+
return { status: 'compiled', rule: ruleResult.rule, trace };
|
|
498
652
|
}
|
|
499
|
-
// ── Pipeline 2: LLM compilation
|
|
500
|
-
// The compilerPrompt (ast-grep
|
|
501
|
-
//
|
|
502
|
-
// a session. Pass it as systemPrompt
|
|
503
|
-
// (mmnto/totem#1291 Phase 3). The user
|
|
504
|
-
//
|
|
653
|
+
// ── Pipeline 2 / Layer 3: LLM compilation with verify-retry loop ──
|
|
654
|
+
// ADR-088 Phase 1 (mmnto-ai/totem#1479). The compilerPrompt (ast-grep
|
|
655
|
+
// manual + few-shot examples, ~50KB) is the persistent system context —
|
|
656
|
+
// same bytes across every call within a session. Pass it as systemPrompt
|
|
657
|
+
// so the orchestrator can cache it (mmnto/totem#1291 Phase 3). The user
|
|
658
|
+
// prompt carries only the per-lesson body, the optional telemetry
|
|
659
|
+
// directive (per --upgrade target, not cacheable), and on retries the
|
|
660
|
+
// prior-attempt feedback block.
|
|
505
661
|
//
|
|
506
|
-
//
|
|
507
|
-
//
|
|
508
|
-
//
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
662
|
+
// Retry semantics:
|
|
663
|
+
// - On smoke-gate zero-match, rebuild the user prompt with a
|
|
664
|
+
// "Previous Attempt Failed Verification" section that names the
|
|
665
|
+
// failed pattern and the badExample it could not match, then call
|
|
666
|
+
// the LLM again. Up to MAX_VERIFY_ATTEMPTS total attempts.
|
|
667
|
+
// - Security context (deps.securityContext === true or the LLM-emitted
|
|
668
|
+
// rule declared `immutable: true`) disables retry: a failing verify
|
|
669
|
+
// rejects outright with reasonCode 'security-rule-rejected' per
|
|
670
|
+
// ADR-088 Decision 3 (zero tolerance).
|
|
671
|
+
// - On attempt exhaustion, fall through to Layer 4 with reasonCode
|
|
672
|
+
// 'verify-retry-exhausted'. The lesson is recorded as skipped
|
|
673
|
+
// with a machine-readable reason so totem doctor and downstream
|
|
674
|
+
// tooling can distinguish this from 'out-of-scope'.
|
|
675
|
+
const MAX_VERIFY_ATTEMPTS = 3;
|
|
676
|
+
let previousFailure = null;
|
|
677
|
+
for (let attempt = 1; attempt <= MAX_VERIFY_ATTEMPTS; attempt++) {
|
|
678
|
+
const userPromptParts = [];
|
|
679
|
+
if (deps.telemetryPrefix) {
|
|
680
|
+
userPromptParts.push('## Telemetry-Driven Refinement Directive', deps.telemetryPrefix);
|
|
681
|
+
}
|
|
682
|
+
if (previousFailure) {
|
|
683
|
+
userPromptParts.push('## Previous Attempt Failed Verification', buildRetryDirective(previousFailure, attempt, MAX_VERIFY_ATTEMPTS));
|
|
684
|
+
}
|
|
685
|
+
userPromptParts.push('## Lesson to Compile', `Heading: ${lesson.heading}`, lesson.body);
|
|
686
|
+
const userPrompt = userPromptParts.join('\n\n');
|
|
687
|
+
const response = await runOrchestrator(userPrompt, compilerPrompt);
|
|
688
|
+
if (response == null) {
|
|
689
|
+
trace.push({ layer: 3, action: 'result', outcome: 'noop' });
|
|
690
|
+
return { status: 'noop', trace };
|
|
691
|
+
}
|
|
692
|
+
const parsed = parseCompilerResponse(response);
|
|
693
|
+
if (!parsed) {
|
|
694
|
+
callbacks?.onWarn?.(lesson.heading, 'Failed to parse LLM response — skipping');
|
|
695
|
+
trace.push({
|
|
696
|
+
layer: 3,
|
|
697
|
+
action: 'result',
|
|
698
|
+
outcome: 'skipped',
|
|
699
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
700
|
+
});
|
|
701
|
+
return {
|
|
702
|
+
status: 'skipped',
|
|
703
|
+
hash: lesson.hash,
|
|
704
|
+
reason: 'Failed to parse LLM response',
|
|
705
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
706
|
+
trace,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
if (!parsed.compilable) {
|
|
710
|
+
callbacks?.onDim?.(lesson.heading, 'Not compilable (conceptual/architectural) — skipping');
|
|
711
|
+
trace.push({
|
|
712
|
+
layer: 3,
|
|
713
|
+
action: 'result',
|
|
714
|
+
outcome: 'skipped',
|
|
715
|
+
reasonCode: 'out-of-scope',
|
|
716
|
+
});
|
|
717
|
+
return {
|
|
718
|
+
status: 'skipped',
|
|
719
|
+
hash: lesson.hash,
|
|
720
|
+
reason: parsed.reason,
|
|
721
|
+
reasonCode: 'out-of-scope',
|
|
722
|
+
trace,
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
// Only include `patternHash` when the parsed output actually carries a
|
|
726
|
+
// pattern. Hashing the `(pattern unavailable)` fallback would make a
|
|
727
|
+
// failed generation look identical to a successful one in verbose output.
|
|
728
|
+
const currentHasPattern = hasExtractablePattern(parsed);
|
|
729
|
+
trace.push({
|
|
730
|
+
layer: 3,
|
|
731
|
+
action: 'generate',
|
|
732
|
+
outcome: `attempt-${attempt}`,
|
|
733
|
+
...(currentHasPattern ? { patternHash: hashPattern(extractPatternString(parsed)) } : {}),
|
|
734
|
+
});
|
|
735
|
+
// Smoke gate enforcement lives in buildCompiledRule (mmnto-ai/totem#1408).
|
|
736
|
+
// Rules without a badExample, or whose badExample fails to match the
|
|
737
|
+
// pattern, come back with rule === null and a rejectReason. Classify the
|
|
738
|
+
// outcome into one of four buckets:
|
|
739
|
+
// success — rule built AND Example Hit/Miss verify passed
|
|
740
|
+
// retry-eligible — smoke-gate zero-match OR verifyRuleExamples failure
|
|
741
|
+
// missing-badexample — LLM omitted a required field (structural)
|
|
742
|
+
// validator-failure — invalid regex / ast-grep parse error / self-
|
|
743
|
+
// suppression guard; retrying would produce more
|
|
744
|
+
// invalid patterns, so the lesson stays pending
|
|
745
|
+
const ruleResult = buildCompiledRule(parsed, lesson, existingByHash, {
|
|
746
|
+
enforceSmokeGate: true,
|
|
747
|
+
});
|
|
748
|
+
let retryReason;
|
|
749
|
+
let retrySnippet;
|
|
750
|
+
if (ruleResult.rule) {
|
|
751
|
+
const testResult = verifyRuleExamples(ruleResult.rule, lesson.body);
|
|
752
|
+
if (!testResult || testResult.passed) {
|
|
753
|
+
// ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): flag rules compiled
|
|
754
|
+
// without a non-empty Example Hit as unverified, even when the
|
|
755
|
+
// lesson carried an Example Miss (the miss still got verified).
|
|
756
|
+
if (!exampleHitPresent) {
|
|
757
|
+
ruleResult.rule.unverified = true;
|
|
758
|
+
}
|
|
759
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'MATCH' });
|
|
760
|
+
trace.push({ layer: 3, action: 'result', outcome: 'compiled' });
|
|
761
|
+
return { status: 'compiled', rule: ruleResult.rule, trace };
|
|
762
|
+
}
|
|
763
|
+
// Example Hit/Miss verification failed against the lesson's ground
|
|
764
|
+
// truth. ADR-088 AC: "verifies every LLM-generated pattern against
|
|
765
|
+
// the lesson's Example Hit block. Zero-match triggers a retry."
|
|
766
|
+
retryReason = formatExampleFailure(testResult);
|
|
767
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'example-hit-miss' });
|
|
768
|
+
// The snippet the retry directive should show is the Example Hit
|
|
769
|
+
// line the pattern missed, not the LLM's own badExample (which the
|
|
770
|
+
// pattern did match, since the smoke gate passed). If missedFails
|
|
771
|
+
// is empty but the test still failed (false-positive-only case),
|
|
772
|
+
// fall back to the badExample so the directive has something
|
|
773
|
+
// concrete to anchor on.
|
|
774
|
+
retrySnippet =
|
|
775
|
+
testResult.missedFails.length > 0
|
|
776
|
+
? testResult.missedFails.join('\n')
|
|
777
|
+
: (parsed.badExample ?? '(no snippet available)');
|
|
778
|
+
}
|
|
779
|
+
else {
|
|
780
|
+
const rejectReason = ruleResult.rejectReason ?? 'Unknown error';
|
|
781
|
+
// Missing-badExample is a structural-output failure. Retrying won't
|
|
782
|
+
// teach the LLM to emit a field it just omitted — the compiler
|
|
783
|
+
// system prompt already requires it (mmnto-ai/totem#1409).
|
|
784
|
+
// Short-circuit to skipped with a distinct reasonCode.
|
|
785
|
+
if (rejectReason.includes('missing badExample')) {
|
|
786
|
+
callbacks?.onWarn?.(lesson.heading, `${rejectReason} — skipping`);
|
|
787
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'missing-badexample' });
|
|
788
|
+
trace.push({
|
|
789
|
+
layer: 3,
|
|
790
|
+
action: 'result',
|
|
791
|
+
outcome: 'skipped',
|
|
792
|
+
reasonCode: 'missing-badexample',
|
|
793
|
+
});
|
|
794
|
+
return {
|
|
795
|
+
status: 'skipped',
|
|
796
|
+
hash: lesson.hash,
|
|
797
|
+
reason: rejectReason,
|
|
798
|
+
reasonCode: 'missing-badexample',
|
|
799
|
+
trace,
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
// Only smoke-gate zero-match against the LLM's own badExample is
|
|
803
|
+
// retry-eligible. Validator-level rejections — invalid regex,
|
|
804
|
+
// ast-grep parse errors, self-suppression guards — are terminal.
|
|
805
|
+
// Retrying them produces more invalid patterns and wastes tokens.
|
|
806
|
+
// Record a machine-readable code in the ledger so `totem doctor`
|
|
807
|
+
// and downstream telemetry can distinguish syntax rejections from
|
|
808
|
+
// retry-exhaustion.
|
|
809
|
+
const isZeroMatch = rejectReason.startsWith('smoke gate: zero matches');
|
|
810
|
+
if (!isZeroMatch) {
|
|
811
|
+
callbacks?.onWarn?.(lesson.heading, `${rejectReason} — skipping`);
|
|
812
|
+
const reasonCode = classifyBuildRejectReason(rejectReason);
|
|
813
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'validator-rejected' });
|
|
814
|
+
trace.push({ layer: 3, action: 'result', outcome: 'skipped', reasonCode });
|
|
815
|
+
return {
|
|
816
|
+
status: 'skipped',
|
|
817
|
+
hash: lesson.hash,
|
|
818
|
+
reason: rejectReason,
|
|
819
|
+
reasonCode,
|
|
820
|
+
trace,
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
retryReason = rejectReason;
|
|
824
|
+
retrySnippet = parsed.badExample ?? '(no badExample emitted)';
|
|
825
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'smoke-gate-zero-match' });
|
|
826
|
+
}
|
|
827
|
+
// Shared retry path. Triggered by smoke-gate zero-match OR by
|
|
828
|
+
// Example Hit/Miss verify failure. Both carry a retryReason that
|
|
829
|
+
// threads back into the next LLM attempt's user prompt.
|
|
830
|
+
// ADR-088 Decision 3: security rules zero-tolerance. No retry.
|
|
831
|
+
if (isSecurityContext(deps, ruleResult.rule)) {
|
|
832
|
+
callbacks?.onWarn?.(lesson.heading, `Security rule rejected on verify failure (no retry): ${retryReason}`);
|
|
833
|
+
trace.push({
|
|
834
|
+
layer: 3,
|
|
835
|
+
action: 'result',
|
|
836
|
+
outcome: 'skipped',
|
|
837
|
+
reasonCode: 'security-rule-rejected',
|
|
838
|
+
});
|
|
839
|
+
return {
|
|
840
|
+
status: 'skipped',
|
|
841
|
+
hash: lesson.hash,
|
|
842
|
+
reason: retryReason,
|
|
843
|
+
reasonCode: 'security-rule-rejected',
|
|
844
|
+
trace,
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
if (attempt < MAX_VERIFY_ATTEMPTS) {
|
|
848
|
+
previousFailure = {
|
|
849
|
+
pattern: extractPatternString(parsed),
|
|
850
|
+
snippet: retrySnippet,
|
|
851
|
+
reason: retryReason,
|
|
852
|
+
};
|
|
853
|
+
callbacks?.onDim?.(lesson.heading, `Verify failed (attempt ${attempt}/${MAX_VERIFY_ATTEMPTS}): ${retryReason} — retrying`);
|
|
854
|
+
trace.push({ layer: 3, action: 'retry', outcome: `attempt-${attempt + 1}-scheduled` });
|
|
855
|
+
continue;
|
|
856
|
+
}
|
|
857
|
+
// Attempts exhausted → Layer 4 fallthrough per ADR-088.
|
|
858
|
+
callbacks?.onWarn?.(lesson.heading, `Verify retry exhausted after ${MAX_VERIFY_ATTEMPTS} attempts: ${retryReason} — skipping`);
|
|
859
|
+
trace.push({
|
|
860
|
+
layer: 3,
|
|
861
|
+
action: 'result',
|
|
862
|
+
outcome: 'skipped',
|
|
863
|
+
reasonCode: 'verify-retry-exhausted',
|
|
864
|
+
});
|
|
865
|
+
return {
|
|
866
|
+
status: 'skipped',
|
|
867
|
+
hash: lesson.hash,
|
|
868
|
+
reason: `Verify retry exhausted after ${MAX_VERIFY_ATTEMPTS} attempts: ${retryReason}`,
|
|
869
|
+
reasonCode: 'verify-retry-exhausted',
|
|
870
|
+
trace,
|
|
871
|
+
};
|
|
537
872
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
873
|
+
// Unreachable: every path inside the loop returns. Kept to satisfy the
|
|
874
|
+
// return-type checker without using a non-null assertion dance. Push a
|
|
875
|
+
// terminal result event defensively so any caller relying on the
|
|
876
|
+
// terminal-result invariant still sees one even if the loop ever leaks.
|
|
877
|
+
trace.push({ layer: 3, action: 'result', outcome: 'failed' });
|
|
878
|
+
return { status: 'failed', trace };
|
|
879
|
+
}
|
|
880
|
+
// ─── Retry loop helpers (ADR-088 Layer 3) ──────────
|
|
881
|
+
/**
|
|
882
|
+
* Map a `buildCompiledRule` rejectReason string to a machine-readable
|
|
883
|
+
* `CompileLessonReasonCode`. Substring matching is deliberate: these
|
|
884
|
+
* strings originate inside `compiler.ts` / `compile-lesson.ts` and the
|
|
885
|
+
* tests on this module pin them, so prefix drift surfaces as a test
|
|
886
|
+
* failure rather than a silent reclassification.
|
|
887
|
+
*
|
|
888
|
+
* - 'Missing pattern/message/astQuery/astGrepPattern/astGrepYamlRule'
|
|
889
|
+
* → `'no-pattern-generated'`. The LLM emitted a structured response
|
|
890
|
+
* without the fields needed to build a rule.
|
|
891
|
+
* - 'Rejected regex' / 'Invalid ast-grep pattern' / 'Pattern matches a
|
|
892
|
+
* suppression directive' → `'pattern-syntax-invalid'`. The pattern
|
|
893
|
+
* exists but cannot be parsed / executed / safely run.
|
|
894
|
+
* - 'smoke gate: zero matches' → `'pattern-zero-match'`. The pattern
|
|
895
|
+
* compiled fine but did not fire against the badExample. On the
|
|
896
|
+
* Pipeline 2 retry-loop path this branch is used for retry fuel;
|
|
897
|
+
* the retry-exhausted terminal uses `'verify-retry-exhausted'`.
|
|
898
|
+
* Pipeline 3 does not retry, so the gate rejection lands here
|
|
899
|
+
* directly.
|
|
900
|
+
* - Fallback is `'pattern-syntax-invalid'` so every skipped ledger
|
|
901
|
+
* entry carries a specific code per ADR-088 Layer 4.
|
|
902
|
+
*/
|
|
903
|
+
function classifyBuildRejectReason(rejectReason) {
|
|
904
|
+
if (rejectReason.startsWith('Missing '))
|
|
905
|
+
return 'no-pattern-generated';
|
|
906
|
+
if (rejectReason.startsWith('Rejected regex'))
|
|
907
|
+
return 'pattern-syntax-invalid';
|
|
908
|
+
if (rejectReason.startsWith('Invalid ast-grep pattern'))
|
|
909
|
+
return 'pattern-syntax-invalid';
|
|
910
|
+
if (rejectReason.includes('suppression directive'))
|
|
911
|
+
return 'pattern-syntax-invalid';
|
|
912
|
+
if (rejectReason.startsWith('smoke gate: zero matches'))
|
|
913
|
+
return 'pattern-zero-match';
|
|
914
|
+
return 'pattern-syntax-invalid';
|
|
915
|
+
}
|
|
916
|
+
/**
|
|
917
|
+
* Render the parsed LLM output's pattern as a printable string for the
|
|
918
|
+
* retry-directive prompt. regex and astGrepPattern are string-native; a
|
|
919
|
+
* compound ast-grep rule is JSON-serialized so the LLM can see the
|
|
920
|
+
* structure that produced zero matches.
|
|
921
|
+
*/
|
|
922
|
+
function extractPatternString(parsed) {
|
|
923
|
+
if (typeof parsed.pattern === 'string' && parsed.pattern.length > 0)
|
|
924
|
+
return parsed.pattern;
|
|
925
|
+
if (typeof parsed.astGrepPattern === 'string' && parsed.astGrepPattern.length > 0) {
|
|
926
|
+
return parsed.astGrepPattern;
|
|
542
927
|
}
|
|
543
|
-
|
|
928
|
+
if (parsed.astGrepYamlRule)
|
|
929
|
+
return JSON.stringify(parsed.astGrepYamlRule, null, 2);
|
|
930
|
+
return '(pattern unavailable)';
|
|
931
|
+
}
|
|
932
|
+
/**
|
|
933
|
+
* Whether the parsed compiler output carries a concrete pattern we can hash.
|
|
934
|
+
* Callers emitting a `generate` trace event use this guard so the event
|
|
935
|
+
* records `patternHash` only when the LLM actually produced a pattern; a
|
|
936
|
+
* missing-pattern case emits the event without the field rather than
|
|
937
|
+
* hashing the `(pattern unavailable)` fallback, which would misleadingly
|
|
938
|
+
* look like a successful generation in verbose output.
|
|
939
|
+
*/
|
|
940
|
+
function hasExtractablePattern(parsed) {
|
|
941
|
+
if (typeof parsed.pattern === 'string' && parsed.pattern.length > 0)
|
|
942
|
+
return true;
|
|
943
|
+
if (typeof parsed.astGrepPattern === 'string' && parsed.astGrepPattern.length > 0)
|
|
944
|
+
return true;
|
|
945
|
+
if (parsed.astGrepYamlRule)
|
|
946
|
+
return true;
|
|
947
|
+
return false;
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* Build the "Previous Attempt Failed Verification" user-prompt block that
|
|
951
|
+
* threads the prior-attempt pattern, badExample, and reject reason back to
|
|
952
|
+
* the LLM so it can correct its output.
|
|
953
|
+
*/
|
|
954
|
+
function buildRetryDirective(failure, attempt, maxAttempts) {
|
|
955
|
+
return [
|
|
956
|
+
`This is attempt ${attempt} of ${maxAttempts}. The previous attempt produced a pattern that failed verification.`,
|
|
957
|
+
'',
|
|
958
|
+
'**Previous pattern:**',
|
|
959
|
+
'```',
|
|
960
|
+
failure.pattern,
|
|
961
|
+
'```',
|
|
962
|
+
'',
|
|
963
|
+
'**Code snippet the pattern had to match:**',
|
|
964
|
+
'```',
|
|
965
|
+
failure.snippet,
|
|
966
|
+
'```',
|
|
967
|
+
'',
|
|
968
|
+
`**Failure reason:** ${failure.reason}`,
|
|
969
|
+
'',
|
|
970
|
+
'Generate a corrected pattern that satisfies the requirements. Keep the lesson intent unchanged; only fix the pattern so it triggers correctly.',
|
|
971
|
+
].join('\n');
|
|
544
972
|
}
|
|
545
973
|
//# sourceMappingURL=compile-lesson.js.map
|