@mmnto/totem 1.14.11 → 1.14.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compile-lesson.d.ts +58 -2
- package/dist/compile-lesson.d.ts.map +1 -1
- package/dist/compile-lesson.js +307 -29
- package/dist/compile-lesson.js.map +1 -1
- package/dist/compile-lesson.test.js +491 -19
- package/dist/compile-lesson.test.js.map +1 -1
- package/dist/compiler-schema.d.ts +154 -9
- package/dist/compiler-schema.d.ts.map +1 -1
- package/dist/compiler-schema.js +88 -10
- package/dist/compiler-schema.js.map +1 -1
- package/dist/compiler.d.ts +11 -3
- package/dist/compiler.d.ts.map +1 -1
- package/dist/compiler.js +24 -4
- package/dist/compiler.js.map +1 -1
- package/dist/compiler.test.js +162 -22
- package/dist/compiler.test.js.map +1 -1
- package/dist/config-schema.d.ts +45 -0
- package/dist/config-schema.d.ts.map +1 -1
- package/dist/config-schema.js +21 -0
- package/dist/config-schema.js.map +1 -1
- package/dist/config-schema.test.js +50 -1
- package/dist/config-schema.test.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/rule-metrics.d.ts +40 -0
- package/dist/rule-metrics.d.ts.map +1 -1
- package/dist/rule-metrics.js +28 -0
- package/dist/rule-metrics.js.map +1 -1
- package/dist/rule-metrics.test.js +104 -3
- package/dist/rule-metrics.test.js.map +1 -1
- package/package.json +1 -1
package/dist/compile-lesson.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CompiledRule, CompilerOutput, RegexValidation } from './compiler-schema.js';
|
|
1
|
+
import type { CompiledRule, CompilerOutput, NonCompilableReasonCode, RegexValidation } from './compiler-schema.js';
|
|
2
2
|
import type { RuleTestResult } from './rule-tester.js';
|
|
3
3
|
export interface LessonInput {
|
|
4
4
|
index: number;
|
|
@@ -11,20 +11,59 @@ export interface LessonInput {
|
|
|
11
11
|
* downstream consumers (totem doctor, Layer 4 fallthrough reporting per ADR-088)
|
|
12
12
|
* can distinguish why a lesson produced no rule without string-matching
|
|
13
13
|
* human-readable messages.
|
|
14
|
+
*
|
|
15
|
+
* mmnto-ai/totem#1481 aligned this internal type 1:1 with the persisted
|
|
16
|
+
* `NonCompilableReasonCode` enum so ledger writers can pass the code through
|
|
17
|
+
* without a mapping table. `'non-compilable'` renamed to `'out-of-scope'`,
|
|
18
|
+
* `'security-verify-rejected'` renamed to `'security-rule-rejected'`, and
|
|
19
|
+
* four producer-facing codes joined: `'no-pattern-generated'`,
|
|
20
|
+
* `'pattern-syntax-invalid'`, `'pattern-zero-match'`, `'no-pattern-found'`.
|
|
21
|
+
* Fresh compile runs MUST NOT emit `'legacy-unknown'`; that sentinel exists
|
|
22
|
+
* solely for migrating pre-#1481 2-tuples.
|
|
23
|
+
*/
|
|
24
|
+
export type CompileLessonReasonCode = Exclude<NonCompilableReasonCode, 'legacy-unknown'>;
|
|
25
|
+
/**
|
|
26
|
+
* Single event inside a lesson's compile pipeline. Appended to a per-lesson
|
|
27
|
+
* `trace` array on every pipeline step (generate / verify / retry / result)
|
|
28
|
+
* and surfaced via `CompileLessonResult.trace` for the CLI `--verbose`
|
|
29
|
+
* renderer (mmnto-ai/totem#1482).
|
|
30
|
+
*
|
|
31
|
+
* `layer` numbers align with the ADR-088 staging (1 = manual, 2 = example-
|
|
32
|
+
* based, 3 = Layer 3 LLM with verify-retry). Consumers MUST tolerate unknown
|
|
33
|
+
* layer numbers so a future ADR-088 phase (dedicated Layer 1 / Layer 2
|
|
34
|
+
* telemetry) can emit without breaking the renderer.
|
|
35
|
+
*
|
|
36
|
+
* `patternHash` is a stable 16-character sha256 prefix of the emitted
|
|
37
|
+
* pattern, included only on `generate` events. Callers use it to correlate
|
|
38
|
+
* retries ("this retry produced the same pattern") without forwarding the
|
|
39
|
+
* pattern string itself.
|
|
40
|
+
*
|
|
41
|
+
* `reasonCode` is only set on the terminal `result` event when the lesson
|
|
42
|
+
* skipped. A compiled or failed lesson omits the field.
|
|
14
43
|
*/
|
|
15
|
-
export
|
|
44
|
+
export interface LayerTraceEvent {
|
|
45
|
+
layer: number;
|
|
46
|
+
action: 'generate' | 'verify' | 'retry' | 'result';
|
|
47
|
+
outcome: string;
|
|
48
|
+
patternHash?: string;
|
|
49
|
+
reasonCode?: Exclude<NonCompilableReasonCode, 'legacy-unknown'>;
|
|
50
|
+
}
|
|
16
51
|
export type CompileLessonResult = {
|
|
17
52
|
status: 'compiled';
|
|
18
53
|
rule: CompiledRule;
|
|
54
|
+
trace?: LayerTraceEvent[];
|
|
19
55
|
} | {
|
|
20
56
|
status: 'skipped';
|
|
21
57
|
hash: string;
|
|
22
58
|
reason?: string;
|
|
23
59
|
reasonCode: CompileLessonReasonCode;
|
|
60
|
+
trace?: LayerTraceEvent[];
|
|
24
61
|
} | {
|
|
25
62
|
status: 'failed';
|
|
63
|
+
trace?: LayerTraceEvent[];
|
|
26
64
|
} | {
|
|
27
65
|
status: 'noop';
|
|
66
|
+
trace?: LayerTraceEvent[];
|
|
28
67
|
};
|
|
29
68
|
export interface CompileLessonCallbacks {
|
|
30
69
|
onWarn?: (heading: string, message: string) => void;
|
|
@@ -160,6 +199,23 @@ export declare function formatExampleFailure(result: RuleTestResult): string;
|
|
|
160
199
|
* Returns { rule, rejectReason } so callers can report why a pattern was rejected.
|
|
161
200
|
*/
|
|
162
201
|
export declare function buildManualRule(lesson: LessonInput, existingByHash: Map<string, CompiledRule>): BuildRuleResult;
|
|
202
|
+
/**
|
|
203
|
+
* Security-context signal for the missing-Example-Hit check. Either the
|
|
204
|
+
* compile orchestrator asserts the pack is security-scoped
|
|
205
|
+
* (`deps.securityContext === true`) OR the rule under construction already
|
|
206
|
+
* carries `immutable: true` (set by the pack manifest, ADR-089). Both
|
|
207
|
+
* paths trigger the zero-tolerance reject per ADR-088 Decision 3.
|
|
208
|
+
*
|
|
209
|
+
* The LLM-emitted `CompilerOutput` does not currently carry an `immutable`
|
|
210
|
+
* field (packs set it at pack-merge time), so the second signal only
|
|
211
|
+
* engages on the Pipeline 1 manual-rule path where `buildManualRule`
|
|
212
|
+
* could synthesize an immutable rule directly. A future change that
|
|
213
|
+
* threads `immutable` through `CompilerOutput` can wire Pipeline 2/3
|
|
214
|
+
* into this helper without touching the call sites.
|
|
215
|
+
*/
|
|
216
|
+
export declare function isSecurityContext(deps: CompileLessonDeps, rule?: {
|
|
217
|
+
immutable?: boolean;
|
|
218
|
+
} | null): boolean;
|
|
163
219
|
/**
|
|
164
220
|
* Compile a single lesson into a rule.
|
|
165
221
|
* Handles both manual patterns (zero LLM) and LLM-compiled patterns.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compile-lesson.d.ts","sourceRoot":"","sources":["../src/compile-lesson.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"compile-lesson.d.ts","sourceRoot":"","sources":["../src/compile-lesson.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,YAAY,EACZ,cAAc,EACd,uBAAuB,EACvB,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAM9B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAKvD,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,uBAAuB,GAAG,OAAO,CAAC,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;AAEzF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,UAAU,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;CACjE;AAED,MAAM,MAAM,mBAAmB,GAC3B;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAA;CAAE,GACrE;IACE,MAAM,EAAE,SAAS,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,uBAAuB,CAAC;IACpC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;CAC3B,GACD;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAA;CAAE,GAC/C;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAA;CAAE,CAAC;AAOlD,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AAED,MAAM,WAAW,iBAAiB;IAChC,qBAAqB,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,cAAc,GAAG,IAAI,CAAC;IACnE;;;;;;;;;;;OAWG;IACH,eAAe,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACxF,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC1C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,+FAA+F;IAC/F,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;OAUG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAID;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,eAAe,CA8GjG;AAiBD;;;;;;GAMG;AACH,MAAM,WAAW,wBAAwB;IACvC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EACzC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACzC,OAAO,GAAE,wBAA6B,GACrC,eAAe,CAgJjB;AAID,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,YAAY,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,YAAY,GAAG,MAAM,CAchE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAc1F;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAanE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,WAAW,EACnB,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GACxC,eAAe,CAgEjB;AAiBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,IAAI,GACpC,OAAO,CAIT;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,WAAW,EACnB,cAAc,EAAE,MAAM,EACtB,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC,mBAAmB,CAAC,CAkd9B"}
|
package/dist/compile-lesson.js
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { Lang, parse } from '@ast-grep/napi';
|
|
2
3
|
import { runSmokeGate } from './compile-smoke-gate.js';
|
|
3
4
|
import { engineFields, sanitizeFileGlobs, validateRegex } from './compiler.js';
|
|
4
5
|
import { extractBadGoodSnippets, extractManualPattern, extractRuleExamples, } from './lesson-pattern.js';
|
|
5
6
|
import { testRule } from './rule-tester.js';
|
|
7
|
+
/** Produce a stable short hash of a pattern string for trace correlation. */
|
|
8
|
+
function hashPattern(pattern) {
|
|
9
|
+
return createHash('sha256').update(pattern).digest('hex').slice(0, 16);
|
|
10
|
+
}
|
|
6
11
|
// ─── ast-grep pattern validation ───────────────────
|
|
7
12
|
/**
|
|
8
13
|
* Compile-time validation for ast-grep patterns (#1062, #1339).
|
|
@@ -410,6 +415,40 @@ export function buildManualRule(lesson, existingByHash) {
|
|
|
410
415
|
};
|
|
411
416
|
}
|
|
412
417
|
// ─── Single-lesson compilation ──────────────────────
|
|
418
|
+
/**
|
|
419
|
+
* Check whether the lesson body carries a non-empty Example Hit block.
|
|
420
|
+
* ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): rules compiled without an
|
|
421
|
+
* Example Hit ship as `unverified: true` (non-security) or fail outright
|
|
422
|
+
* (security). Empty code fences like ```ts\n``` are treated as absent —
|
|
423
|
+
* `trim()` is applied before the length check.
|
|
424
|
+
*/
|
|
425
|
+
function hasExampleHits(body) {
|
|
426
|
+
const examples = extractRuleExamples(body);
|
|
427
|
+
if (!examples)
|
|
428
|
+
return false;
|
|
429
|
+
return examples.hits.some((line) => line.trim().length > 0);
|
|
430
|
+
}
|
|
431
|
+
/**
|
|
432
|
+
* Security-context signal for the missing-Example-Hit check. Either the
|
|
433
|
+
* compile orchestrator asserts the pack is security-scoped
|
|
434
|
+
* (`deps.securityContext === true`) OR the rule under construction already
|
|
435
|
+
* carries `immutable: true` (set by the pack manifest, ADR-089). Both
|
|
436
|
+
* paths trigger the zero-tolerance reject per ADR-088 Decision 3.
|
|
437
|
+
*
|
|
438
|
+
* The LLM-emitted `CompilerOutput` does not currently carry an `immutable`
|
|
439
|
+
* field (packs set it at pack-merge time), so the second signal only
|
|
440
|
+
* engages on the Pipeline 1 manual-rule path where `buildManualRule`
|
|
441
|
+
* could synthesize an immutable rule directly. A future change that
|
|
442
|
+
* threads `immutable` through `CompilerOutput` can wire Pipeline 2/3
|
|
443
|
+
* into this helper without touching the call sites.
|
|
444
|
+
*/
|
|
445
|
+
export function isSecurityContext(deps, rule) {
|
|
446
|
+
if (deps.securityContext === true)
|
|
447
|
+
return true;
|
|
448
|
+
if (rule?.immutable === true)
|
|
449
|
+
return true;
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
413
452
|
/**
|
|
414
453
|
* Compile a single lesson into a rule.
|
|
415
454
|
* Handles both manual patterns (zero LLM) and LLM-compiled patterns.
|
|
@@ -417,21 +456,79 @@ export function buildManualRule(lesson, existingByHash) {
|
|
|
417
456
|
*/
|
|
418
457
|
export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
419
458
|
const { parseCompilerResponse, runOrchestrator, existingByHash, callbacks } = deps;
|
|
459
|
+
const exampleHitPresent = hasExampleHits(lesson.body);
|
|
460
|
+
// Trace buffer (mmnto-ai/totem#1482). Populated unconditionally across all
|
|
461
|
+
// three pipelines so the CLI can choose to render it under --verbose
|
|
462
|
+
// without a per-call gate. Cost of empty-or-small arrays per lesson is
|
|
463
|
+
// negligible; gating would fork the test surface.
|
|
464
|
+
const trace = [];
|
|
420
465
|
// ── Pipeline 1: Manual pattern (zero LLM) ────────
|
|
421
466
|
const manualResult = buildManualRule(lesson, existingByHash);
|
|
422
467
|
if (manualResult.rule) {
|
|
468
|
+
// ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): security-scoped manual
|
|
469
|
+
// rules without an Example Hit are rejected outright.
|
|
470
|
+
if (!exampleHitPresent && isSecurityContext(deps, manualResult.rule)) {
|
|
471
|
+
const reason = 'Security rule missing Example Hit block (manual)';
|
|
472
|
+
callbacks?.onWarn?.(lesson.heading, `${reason} — rejecting`);
|
|
473
|
+
trace.push({
|
|
474
|
+
layer: 1,
|
|
475
|
+
action: 'result',
|
|
476
|
+
outcome: 'skipped',
|
|
477
|
+
reasonCode: 'security-rule-rejected',
|
|
478
|
+
});
|
|
479
|
+
return {
|
|
480
|
+
status: 'skipped',
|
|
481
|
+
hash: lesson.hash,
|
|
482
|
+
reason,
|
|
483
|
+
reasonCode: 'security-rule-rejected',
|
|
484
|
+
trace,
|
|
485
|
+
};
|
|
486
|
+
}
|
|
423
487
|
const testResult = verifyRuleExamples(manualResult.rule, lesson.body);
|
|
424
488
|
if (testResult && !testResult.passed) {
|
|
489
|
+
// A manual rule that failed its own inline examples is authoring
|
|
490
|
+
// error territory. Keep the existing 'failed' contract so the
|
|
491
|
+
// lesson stays pending rather than landing in nonCompilable.
|
|
492
|
+
// Pre-#1480 `verifyRuleExamples` fired on any hit or miss; the
|
|
493
|
+
// `hasExampleHits` check guards the unverified flag below without
|
|
494
|
+
// changing the verify gate itself.
|
|
425
495
|
callbacks?.onWarn?.(lesson.heading, formatExampleFailure(testResult));
|
|
426
|
-
|
|
496
|
+
trace.push({ layer: 1, action: 'result', outcome: 'failed' });
|
|
497
|
+
return { status: 'failed', trace };
|
|
427
498
|
}
|
|
428
|
-
|
|
499
|
+
if (!exampleHitPresent) {
|
|
500
|
+
manualResult.rule.unverified = true;
|
|
501
|
+
}
|
|
502
|
+
trace.push({ layer: 1, action: 'result', outcome: 'compiled' });
|
|
503
|
+
return { status: 'compiled', rule: manualResult.rule, trace };
|
|
429
504
|
}
|
|
430
505
|
if (manualResult.rejectReason) {
|
|
431
506
|
callbacks?.onWarn?.(lesson.heading, manualResult.rejectReason);
|
|
432
|
-
|
|
507
|
+
trace.push({ layer: 1, action: 'result', outcome: 'failed' });
|
|
508
|
+
return { status: 'failed', trace };
|
|
433
509
|
}
|
|
434
510
|
// manualResult.rule === null && no rejectReason → no manual pattern, proceed to Pipeline 3 or 2
|
|
511
|
+
// ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): security rules that enter
|
|
512
|
+
// the LLM path with no Example Hit short-circuit before the orchestrator
|
|
513
|
+
// call. Compile has no ground truth to verify against, and zero-tolerance
|
|
514
|
+
// per Decision 3 leaves nothing to retry.
|
|
515
|
+
if (!exampleHitPresent && deps.securityContext === true) {
|
|
516
|
+
const reason = 'Security rule missing Example Hit block';
|
|
517
|
+
callbacks?.onWarn?.(lesson.heading, `${reason} — rejecting`);
|
|
518
|
+
trace.push({
|
|
519
|
+
layer: 3,
|
|
520
|
+
action: 'result',
|
|
521
|
+
outcome: 'skipped',
|
|
522
|
+
reasonCode: 'security-rule-rejected',
|
|
523
|
+
});
|
|
524
|
+
return {
|
|
525
|
+
status: 'skipped',
|
|
526
|
+
hash: lesson.hash,
|
|
527
|
+
reason,
|
|
528
|
+
reasonCode: 'security-rule-rejected',
|
|
529
|
+
trace,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
435
532
|
// ── Pipeline 3: Example-based compilation (Bad/Good snippets) ──
|
|
436
533
|
const snippets = extractBadGoodSnippets(lesson.body);
|
|
437
534
|
if (snippets) {
|
|
@@ -455,22 +552,53 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
455
552
|
lesson.body,
|
|
456
553
|
].join('\n');
|
|
457
554
|
const response = await runOrchestrator(userPrompt, systemPrompt);
|
|
458
|
-
if (response == null)
|
|
459
|
-
|
|
555
|
+
if (response == null) {
|
|
556
|
+
trace.push({ layer: 2, action: 'result', outcome: 'noop' });
|
|
557
|
+
return { status: 'noop', trace };
|
|
558
|
+
}
|
|
460
559
|
const parsed = parseCompilerResponse(response);
|
|
461
560
|
if (!parsed) {
|
|
462
561
|
callbacks?.onWarn?.(lesson.heading, 'Pipeline 3: failed to parse LLM response — skipping');
|
|
463
|
-
|
|
562
|
+
trace.push({
|
|
563
|
+
layer: 2,
|
|
564
|
+
action: 'result',
|
|
565
|
+
outcome: 'skipped',
|
|
566
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
567
|
+
});
|
|
568
|
+
return {
|
|
569
|
+
status: 'skipped',
|
|
570
|
+
hash: lesson.hash,
|
|
571
|
+
reason: 'Pipeline 3: failed to parse LLM response',
|
|
572
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
573
|
+
trace,
|
|
574
|
+
};
|
|
464
575
|
}
|
|
465
576
|
if (!parsed.compilable) {
|
|
466
577
|
callbacks?.onDim?.(lesson.heading, 'Pipeline 3: not compilable — skipping');
|
|
578
|
+
trace.push({
|
|
579
|
+
layer: 2,
|
|
580
|
+
action: 'result',
|
|
581
|
+
outcome: 'skipped',
|
|
582
|
+
reasonCode: 'out-of-scope',
|
|
583
|
+
});
|
|
467
584
|
return {
|
|
468
585
|
status: 'skipped',
|
|
469
586
|
hash: lesson.hash,
|
|
470
587
|
reason: parsed.reason,
|
|
471
|
-
reasonCode: '
|
|
588
|
+
reasonCode: 'out-of-scope',
|
|
589
|
+
trace,
|
|
472
590
|
};
|
|
473
591
|
}
|
|
592
|
+
// Only include `patternHash` when the parsed output actually carries a
|
|
593
|
+
// pattern. Hashing the `(pattern unavailable)` fallback would make a
|
|
594
|
+
// failed generation look identical to a successful one in verbose output.
|
|
595
|
+
const pipeline3HasPattern = hasExtractablePattern(parsed);
|
|
596
|
+
trace.push({
|
|
597
|
+
layer: 2,
|
|
598
|
+
action: 'generate',
|
|
599
|
+
outcome: 'produced',
|
|
600
|
+
...(pipeline3HasPattern ? { patternHash: hashPattern(extractPatternString(parsed)) } : {}),
|
|
601
|
+
});
|
|
474
602
|
// mmnto/totem#1408: Pipeline 3 reuses its Bad snippet as the smoke-gate
|
|
475
603
|
// target. The LLM may or may not echo the snippet back in parsed.badExample;
|
|
476
604
|
// the override guarantees the gate has something to work with regardless.
|
|
@@ -479,8 +607,18 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
479
607
|
badExampleOverride: snippets.bad.join('\n'),
|
|
480
608
|
});
|
|
481
609
|
if (!ruleResult.rule) {
|
|
482
|
-
|
|
483
|
-
|
|
610
|
+
const rejectReason = ruleResult.rejectReason ?? 'Unknown error';
|
|
611
|
+
callbacks?.onWarn?.(lesson.heading, `Pipeline 3: ${rejectReason} — skipping`);
|
|
612
|
+
const reasonCode = classifyBuildRejectReason(rejectReason);
|
|
613
|
+
trace.push({ layer: 2, action: 'verify', outcome: 'rejected' });
|
|
614
|
+
trace.push({ layer: 2, action: 'result', outcome: 'skipped', reasonCode });
|
|
615
|
+
return {
|
|
616
|
+
status: 'skipped',
|
|
617
|
+
hash: lesson.hash,
|
|
618
|
+
reason: `Pipeline 3: ${rejectReason}`,
|
|
619
|
+
reasonCode,
|
|
620
|
+
trace,
|
|
621
|
+
};
|
|
484
622
|
}
|
|
485
623
|
// Self-verify: at least one Bad line should trigger, no Good line should trigger
|
|
486
624
|
const virtualPath = deriveVirtualFilePath(ruleResult.rule);
|
|
@@ -497,9 +635,20 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
497
635
|
const badCaught = snippets.bad.length - testResult.missedFails.length;
|
|
498
636
|
if (badCaught === 0 || testResult.falsePositives.length > 0) {
|
|
499
637
|
callbacks?.onWarn?.(lesson.heading, 'Pipeline 3: generated rule failed self-verification against Bad/Good snippets — skipping');
|
|
500
|
-
|
|
638
|
+
trace.push({ layer: 2, action: 'verify', outcome: 'self-test-failed' });
|
|
639
|
+
trace.push({ layer: 2, action: 'result', outcome: 'failed' });
|
|
640
|
+
return { status: 'failed', trace };
|
|
501
641
|
}
|
|
502
|
-
|
|
642
|
+
trace.push({ layer: 2, action: 'verify', outcome: 'passed' });
|
|
643
|
+
trace.push({ layer: 2, action: 'result', outcome: 'compiled' });
|
|
644
|
+
// Pipeline 3 rules are example-based — they always ship with a Bad snippet
|
|
645
|
+
// by construction. Flag unverified only when the lesson body itself omits
|
|
646
|
+
// the canonical Example Hit block, keeping the signal aligned with the
|
|
647
|
+
// lesson-level ground-truth rather than the Pipeline 3 self-test.
|
|
648
|
+
if (!exampleHitPresent) {
|
|
649
|
+
ruleResult.rule.unverified = true;
|
|
650
|
+
}
|
|
651
|
+
return { status: 'compiled', rule: ruleResult.rule, trace };
|
|
503
652
|
}
|
|
504
653
|
// ── Pipeline 2 / Layer 3: LLM compilation with verify-retry loop ──
|
|
505
654
|
// ADR-088 Phase 1 (mmnto-ai/totem#1479). The compilerPrompt (ast-grep
|
|
@@ -515,14 +664,14 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
515
664
|
// "Previous Attempt Failed Verification" section that names the
|
|
516
665
|
// failed pattern and the badExample it could not match, then call
|
|
517
666
|
// the LLM again. Up to MAX_VERIFY_ATTEMPTS total attempts.
|
|
518
|
-
// - Security context (deps.securityContext === true
|
|
519
|
-
//
|
|
520
|
-
// 'security-
|
|
521
|
-
// tolerance).
|
|
667
|
+
// - Security context (deps.securityContext === true or the LLM-emitted
|
|
668
|
+
// rule declared `immutable: true`) disables retry: a failing verify
|
|
669
|
+
// rejects outright with reasonCode 'security-rule-rejected' per
|
|
670
|
+
// ADR-088 Decision 3 (zero tolerance).
|
|
522
671
|
// - On attempt exhaustion, fall through to Layer 4 with reasonCode
|
|
523
672
|
// 'verify-retry-exhausted'. The lesson is recorded as skipped
|
|
524
673
|
// with a machine-readable reason so totem doctor and downstream
|
|
525
|
-
// tooling can distinguish this from '
|
|
674
|
+
// tooling can distinguish this from 'out-of-scope'.
|
|
526
675
|
const MAX_VERIFY_ATTEMPTS = 3;
|
|
527
676
|
let previousFailure = null;
|
|
528
677
|
for (let attempt = 1; attempt <= MAX_VERIFY_ATTEMPTS; attempt++) {
|
|
@@ -536,22 +685,53 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
536
685
|
userPromptParts.push('## Lesson to Compile', `Heading: ${lesson.heading}`, lesson.body);
|
|
537
686
|
const userPrompt = userPromptParts.join('\n\n');
|
|
538
687
|
const response = await runOrchestrator(userPrompt, compilerPrompt);
|
|
539
|
-
if (response == null)
|
|
540
|
-
|
|
688
|
+
if (response == null) {
|
|
689
|
+
trace.push({ layer: 3, action: 'result', outcome: 'noop' });
|
|
690
|
+
return { status: 'noop', trace };
|
|
691
|
+
}
|
|
541
692
|
const parsed = parseCompilerResponse(response);
|
|
542
693
|
if (!parsed) {
|
|
543
694
|
callbacks?.onWarn?.(lesson.heading, 'Failed to parse LLM response — skipping');
|
|
544
|
-
|
|
695
|
+
trace.push({
|
|
696
|
+
layer: 3,
|
|
697
|
+
action: 'result',
|
|
698
|
+
outcome: 'skipped',
|
|
699
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
700
|
+
});
|
|
701
|
+
return {
|
|
702
|
+
status: 'skipped',
|
|
703
|
+
hash: lesson.hash,
|
|
704
|
+
reason: 'Failed to parse LLM response',
|
|
705
|
+
reasonCode: 'pattern-syntax-invalid',
|
|
706
|
+
trace,
|
|
707
|
+
};
|
|
545
708
|
}
|
|
546
709
|
if (!parsed.compilable) {
|
|
547
710
|
callbacks?.onDim?.(lesson.heading, 'Not compilable (conceptual/architectural) — skipping');
|
|
711
|
+
trace.push({
|
|
712
|
+
layer: 3,
|
|
713
|
+
action: 'result',
|
|
714
|
+
outcome: 'skipped',
|
|
715
|
+
reasonCode: 'out-of-scope',
|
|
716
|
+
});
|
|
548
717
|
return {
|
|
549
718
|
status: 'skipped',
|
|
550
719
|
hash: lesson.hash,
|
|
551
720
|
reason: parsed.reason,
|
|
552
|
-
reasonCode: '
|
|
721
|
+
reasonCode: 'out-of-scope',
|
|
722
|
+
trace,
|
|
553
723
|
};
|
|
554
724
|
}
|
|
725
|
+
// Only include `patternHash` when the parsed output actually carries a
|
|
726
|
+
// pattern. Hashing the `(pattern unavailable)` fallback would make a
|
|
727
|
+
// failed generation look identical to a successful one in verbose output.
|
|
728
|
+
const currentHasPattern = hasExtractablePattern(parsed);
|
|
729
|
+
trace.push({
|
|
730
|
+
layer: 3,
|
|
731
|
+
action: 'generate',
|
|
732
|
+
outcome: `attempt-${attempt}`,
|
|
733
|
+
...(currentHasPattern ? { patternHash: hashPattern(extractPatternString(parsed)) } : {}),
|
|
734
|
+
});
|
|
555
735
|
// Smoke gate enforcement lives in buildCompiledRule (mmnto-ai/totem#1408).
|
|
556
736
|
// Rules without a badExample, or whose badExample fails to match the
|
|
557
737
|
// pattern, come back with rule === null and a rejectReason. Classify the
|
|
@@ -570,12 +750,21 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
570
750
|
if (ruleResult.rule) {
|
|
571
751
|
const testResult = verifyRuleExamples(ruleResult.rule, lesson.body);
|
|
572
752
|
if (!testResult || testResult.passed) {
|
|
573
|
-
|
|
753
|
+
// ADR-088 Phase 1 Layer 3 (mmnto-ai/totem#1480): flag rules compiled
|
|
754
|
+
// without a non-empty Example Hit as unverified, even when the
|
|
755
|
+
// lesson carried an Example Miss (the miss still got verified).
|
|
756
|
+
if (!exampleHitPresent) {
|
|
757
|
+
ruleResult.rule.unverified = true;
|
|
758
|
+
}
|
|
759
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'MATCH' });
|
|
760
|
+
trace.push({ layer: 3, action: 'result', outcome: 'compiled' });
|
|
761
|
+
return { status: 'compiled', rule: ruleResult.rule, trace };
|
|
574
762
|
}
|
|
575
763
|
// Example Hit/Miss verification failed against the lesson's ground
|
|
576
764
|
// truth. ADR-088 AC: "verifies every LLM-generated pattern against
|
|
577
765
|
// the lesson's Example Hit block. Zero-match triggers a retry."
|
|
578
766
|
retryReason = formatExampleFailure(testResult);
|
|
767
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'example-hit-miss' });
|
|
579
768
|
// The snippet the retry directive should show is the Example Hit
|
|
580
769
|
// line the pattern missed, not the LLM's own badExample (which the
|
|
581
770
|
// pattern did match, since the smoke gate passed). If missedFails
|
|
@@ -595,38 +784,64 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
595
784
|
// Short-circuit to skipped with a distinct reasonCode.
|
|
596
785
|
if (rejectReason.includes('missing badExample')) {
|
|
597
786
|
callbacks?.onWarn?.(lesson.heading, `${rejectReason} — skipping`);
|
|
787
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'missing-badexample' });
|
|
788
|
+
trace.push({
|
|
789
|
+
layer: 3,
|
|
790
|
+
action: 'result',
|
|
791
|
+
outcome: 'skipped',
|
|
792
|
+
reasonCode: 'missing-badexample',
|
|
793
|
+
});
|
|
598
794
|
return {
|
|
599
795
|
status: 'skipped',
|
|
600
796
|
hash: lesson.hash,
|
|
601
797
|
reason: rejectReason,
|
|
602
798
|
reasonCode: 'missing-badexample',
|
|
799
|
+
trace,
|
|
603
800
|
};
|
|
604
801
|
}
|
|
605
802
|
// Only smoke-gate zero-match against the LLM's own badExample is
|
|
606
803
|
// retry-eligible. Validator-level rejections — invalid regex,
|
|
607
804
|
// ast-grep parse errors, self-suppression guards — are terminal.
|
|
608
805
|
// Retrying them produces more invalid patterns and wastes tokens.
|
|
609
|
-
//
|
|
610
|
-
//
|
|
806
|
+
// Record a machine-readable code in the ledger so `totem doctor`
|
|
807
|
+
// and downstream telemetry can distinguish syntax rejections from
|
|
808
|
+
// retry-exhaustion.
|
|
611
809
|
const isZeroMatch = rejectReason.startsWith('smoke gate: zero matches');
|
|
612
810
|
if (!isZeroMatch) {
|
|
613
|
-
callbacks?.onWarn?.(lesson.heading, `${rejectReason} —
|
|
614
|
-
|
|
811
|
+
callbacks?.onWarn?.(lesson.heading, `${rejectReason} — skipping`);
|
|
812
|
+
const reasonCode = classifyBuildRejectReason(rejectReason);
|
|
813
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'validator-rejected' });
|
|
814
|
+
trace.push({ layer: 3, action: 'result', outcome: 'skipped', reasonCode });
|
|
815
|
+
return {
|
|
816
|
+
status: 'skipped',
|
|
817
|
+
hash: lesson.hash,
|
|
818
|
+
reason: rejectReason,
|
|
819
|
+
reasonCode,
|
|
820
|
+
trace,
|
|
821
|
+
};
|
|
615
822
|
}
|
|
616
823
|
retryReason = rejectReason;
|
|
617
824
|
retrySnippet = parsed.badExample ?? '(no badExample emitted)';
|
|
825
|
+
trace.push({ layer: 3, action: 'verify', outcome: 'smoke-gate-zero-match' });
|
|
618
826
|
}
|
|
619
827
|
// Shared retry path. Triggered by smoke-gate zero-match OR by
|
|
620
828
|
// Example Hit/Miss verify failure. Both carry a retryReason that
|
|
621
829
|
// threads back into the next LLM attempt's user prompt.
|
|
622
830
|
// ADR-088 Decision 3: security rules zero-tolerance. No retry.
|
|
623
|
-
if (deps.
|
|
831
|
+
if (isSecurityContext(deps, ruleResult.rule)) {
|
|
624
832
|
callbacks?.onWarn?.(lesson.heading, `Security rule rejected on verify failure (no retry): ${retryReason}`);
|
|
833
|
+
trace.push({
|
|
834
|
+
layer: 3,
|
|
835
|
+
action: 'result',
|
|
836
|
+
outcome: 'skipped',
|
|
837
|
+
reasonCode: 'security-rule-rejected',
|
|
838
|
+
});
|
|
625
839
|
return {
|
|
626
840
|
status: 'skipped',
|
|
627
841
|
hash: lesson.hash,
|
|
628
842
|
reason: retryReason,
|
|
629
|
-
reasonCode: 'security-
|
|
843
|
+
reasonCode: 'security-rule-rejected',
|
|
844
|
+
trace,
|
|
630
845
|
};
|
|
631
846
|
}
|
|
632
847
|
if (attempt < MAX_VERIFY_ATTEMPTS) {
|
|
@@ -636,22 +851,68 @@ export async function compileLesson(lesson, compilerPrompt, deps) {
|
|
|
636
851
|
reason: retryReason,
|
|
637
852
|
};
|
|
638
853
|
callbacks?.onDim?.(lesson.heading, `Verify failed (attempt ${attempt}/${MAX_VERIFY_ATTEMPTS}): ${retryReason} — retrying`);
|
|
854
|
+
trace.push({ layer: 3, action: 'retry', outcome: `attempt-${attempt + 1}-scheduled` });
|
|
639
855
|
continue;
|
|
640
856
|
}
|
|
641
857
|
// Attempts exhausted → Layer 4 fallthrough per ADR-088.
|
|
642
858
|
callbacks?.onWarn?.(lesson.heading, `Verify retry exhausted after ${MAX_VERIFY_ATTEMPTS} attempts: ${retryReason} — skipping`);
|
|
859
|
+
trace.push({
|
|
860
|
+
layer: 3,
|
|
861
|
+
action: 'result',
|
|
862
|
+
outcome: 'skipped',
|
|
863
|
+
reasonCode: 'verify-retry-exhausted',
|
|
864
|
+
});
|
|
643
865
|
return {
|
|
644
866
|
status: 'skipped',
|
|
645
867
|
hash: lesson.hash,
|
|
646
868
|
reason: `Verify retry exhausted after ${MAX_VERIFY_ATTEMPTS} attempts: ${retryReason}`,
|
|
647
869
|
reasonCode: 'verify-retry-exhausted',
|
|
870
|
+
trace,
|
|
648
871
|
};
|
|
649
872
|
}
|
|
650
873
|
// Unreachable: every path inside the loop returns. Kept to satisfy the
|
|
651
|
-
// return-type checker without using a non-null assertion dance.
|
|
652
|
-
|
|
874
|
+
// return-type checker without using a non-null assertion dance. Push a
|
|
875
|
+
// terminal result event defensively so any caller relying on the
|
|
876
|
+
// terminal-result invariant still sees one even if the loop ever leaks.
|
|
877
|
+
trace.push({ layer: 3, action: 'result', outcome: 'failed' });
|
|
878
|
+
return { status: 'failed', trace };
|
|
653
879
|
}
|
|
654
880
|
// ─── Retry loop helpers (ADR-088 Layer 3) ──────────
|
|
881
|
+
/**
|
|
882
|
+
* Map a `buildCompiledRule` rejectReason string to a machine-readable
|
|
883
|
+
* `CompileLessonReasonCode`. Substring matching is deliberate: these
|
|
884
|
+
* strings originate inside `compiler.ts` / `compile-lesson.ts` and the
|
|
885
|
+
* tests on this module pin them, so prefix drift surfaces as a test
|
|
886
|
+
* failure rather than a silent reclassification.
|
|
887
|
+
*
|
|
888
|
+
* - 'Missing pattern/message/astQuery/astGrepPattern/astGrepYamlRule'
|
|
889
|
+
* → `'no-pattern-generated'`. The LLM emitted a structured response
|
|
890
|
+
* without the fields needed to build a rule.
|
|
891
|
+
* - 'Rejected regex' / 'Invalid ast-grep pattern' / 'Pattern matches a
|
|
892
|
+
* suppression directive' → `'pattern-syntax-invalid'`. The pattern
|
|
893
|
+
* exists but cannot be parsed / executed / safely run.
|
|
894
|
+
* - 'smoke gate: zero matches' → `'pattern-zero-match'`. The pattern
|
|
895
|
+
* compiled fine but did not fire against the badExample. On the
|
|
896
|
+
* Pipeline 2 retry-loop path this branch is used for retry fuel;
|
|
897
|
+
* the retry-exhausted terminal uses `'verify-retry-exhausted'`.
|
|
898
|
+
* Pipeline 3 does not retry, so the gate rejection lands here
|
|
899
|
+
* directly.
|
|
900
|
+
* - Fallback is `'pattern-syntax-invalid'` so every skipped ledger
|
|
901
|
+
* entry carries a specific code per ADR-088 Layer 4.
|
|
902
|
+
*/
|
|
903
|
+
function classifyBuildRejectReason(rejectReason) {
|
|
904
|
+
if (rejectReason.startsWith('Missing '))
|
|
905
|
+
return 'no-pattern-generated';
|
|
906
|
+
if (rejectReason.startsWith('Rejected regex'))
|
|
907
|
+
return 'pattern-syntax-invalid';
|
|
908
|
+
if (rejectReason.startsWith('Invalid ast-grep pattern'))
|
|
909
|
+
return 'pattern-syntax-invalid';
|
|
910
|
+
if (rejectReason.includes('suppression directive'))
|
|
911
|
+
return 'pattern-syntax-invalid';
|
|
912
|
+
if (rejectReason.startsWith('smoke gate: zero matches'))
|
|
913
|
+
return 'pattern-zero-match';
|
|
914
|
+
return 'pattern-syntax-invalid';
|
|
915
|
+
}
|
|
655
916
|
/**
|
|
656
917
|
* Render the parsed LLM output's pattern as a printable string for the
|
|
657
918
|
* retry-directive prompt. regex and astGrepPattern are string-native; a
|
|
@@ -668,6 +929,23 @@ function extractPatternString(parsed) {
|
|
|
668
929
|
return JSON.stringify(parsed.astGrepYamlRule, null, 2);
|
|
669
930
|
return '(pattern unavailable)';
|
|
670
931
|
}
|
|
932
|
+
/**
|
|
933
|
+
* Whether the parsed compiler output carries a concrete pattern we can hash.
|
|
934
|
+
* Callers emitting a `generate` trace event use this guard so the event
|
|
935
|
+
* records `patternHash` only when the LLM actually produced a pattern; a
|
|
936
|
+
* missing-pattern case emits the event without the field rather than
|
|
937
|
+
* hashing the `(pattern unavailable)` fallback, which would misleadingly
|
|
938
|
+
* look like a successful generation in verbose output.
|
|
939
|
+
*/
|
|
940
|
+
function hasExtractablePattern(parsed) {
|
|
941
|
+
if (typeof parsed.pattern === 'string' && parsed.pattern.length > 0)
|
|
942
|
+
return true;
|
|
943
|
+
if (typeof parsed.astGrepPattern === 'string' && parsed.astGrepPattern.length > 0)
|
|
944
|
+
return true;
|
|
945
|
+
if (parsed.astGrepYamlRule)
|
|
946
|
+
return true;
|
|
947
|
+
return false;
|
|
948
|
+
}
|
|
671
949
|
/**
|
|
672
950
|
* Build the "Previous Attempt Failed Verification" user-prompt block that
|
|
673
951
|
* threads the prior-attempt pattern, badExample, and reject reason back to
|