@devflow-tools/benchmark 0.17.3 → 0.17.4

package/dist/index.d.ts

@@ -21,4 +21,7 @@ export type { EvidenceQualityMetadata, EvidenceQualityMetric, WilsonInterval95,
 export * from './graders/index.js';
 export { aggregateRepeatedTrials, createTrialReceipt, renderRepeatedTrialReport, } from './repeated-trials.js';
 export type { RepeatedTrial, RepeatedTrialReport, RepeatedTrialTaskReport, } from './repeated-trials.js';
+export { evaluateSemanticFixtures } from './semantic-intent-quality.js';
+export { evaluateSemanticCalibration, evaluateSemanticReleaseGates, wilsonInterval95 } from './semantic-intent-quality.js';
+export type { SemanticIntentCorpus, SemanticIntentExpectedFrame, SemanticIntentFixture, SemanticMutationPermission, SemanticQualityReport, SemanticCalibrationReport, SemanticCalibrationSample, SemanticReleaseGates, } from './semantic-intent-quality.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,YAAY,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAC1E,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,YAAY,EACV,4BAA4B,EAC5B,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,6BAA6B,EAC7B,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,YAAY,EACV,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,+BAA+B,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,YAAY,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAC1E,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,YAAY,EACV,4BAA4B,EAC5B,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,6BAA6B,EAC7B,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,YAAY,EACV,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,+BAA+B,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC3H,YAAY,EACV,oBAAoB,EACpB,2BAA2B,EAC3B,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,8BAA8B,CAAC"}

package/dist/index.js

@@ -11,4 +11,6 @@ export { runEvidenceRuntimeQualityCorpus } from './retrieval-quality-corpus.js';
 export { evidenceQualityMetric } from './evidence-quality-metrics.js';
 export * from './graders/index.js';
 export { aggregateRepeatedTrials, createTrialReceipt, renderRepeatedTrialReport, } from './repeated-trials.js';
+export { evaluateSemanticFixtures } from './semantic-intent-quality.js';
+export { evaluateSemanticCalibration, evaluateSemanticReleaseGates, wilsonInterval95 } from './semantic-intent-quality.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAQ3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAS1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAMtE,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAQ3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAS1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAMtE,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC"}

package/dist/semantic-intent-quality.d.ts

@@ -0,0 +1,82 @@
+import type { CanonicalTaskIntent, RetrievalTaskAction } from '@devflow-tools/sdk';
+export type SemanticMutationPermission = 'explicitly_authorized' | 'explicitly_forbidden' | 'not_requested' | 'conditional' | 'ambiguous';
+export interface SemanticIntentExpectedFrame {
+    primaryIntent: CanonicalTaskIntent;
+    primaryAction: RetrievalTaskAction;
+    mutationPermission: SemanticMutationPermission;
+    memoryDirective: 'save' | 'forget' | 'recall' | 'none' | 'ambiguous';
+    knowledgeKind?: 'api' | 'behavior' | 'migration' | 'error' | 'overview';
+    outOfScope: boolean;
+}
+export interface SemanticIntentFixture {
+    id: string;
+    family: string;
+    partition: 'development' | 'heldout';
+    prompt: string;
+    expected: SemanticIntentExpectedFrame;
+}
+export interface SemanticIntentCorpus {
+    schemaVersion: 'devflow.semantic-intent-corpus.v1';
+    fixtures: SemanticIntentFixture[];
+}
+export interface SemanticQualityReport {
+    schemaVersion: 'devflow.semantic-quality-report.v1';
+    sampleCount: number;
+    exactFrameMatch: number;
+    mutationFalseAccept: {
+        numerator: number;
+        denominator: number;
+        value: number;
+    };
+    mutationFalseReject: {
+        numerator: number;
+        denominator: number;
+        value: number;
+    };
+    prohibitionAccuracy: number;
+    outOfScopeRecall: number;
+    macroF1: number;
+    diagnostics: Array<{
+        fixtureId: string;
+        fields: string[];
+    }>;
+}
+export interface SemanticCalibrationSample {
+    probability: number;
+    outcome: boolean;
+    outOfScope?: boolean;
+    language?: 'zh' | 'en' | 'mixed';
+}
+export interface SemanticCalibrationReport {
+    brierScore: number;
+    expectedCalibrationError: number;
+    wilson95: {
+        lower: number;
+        upper: number;
+    };
+    sampleCount: number;
+}
+export interface SemanticReleaseGates {
+    mutationFalseAccept: boolean;
+    structuralCompatibility: boolean;
+    prohibitionImproved: boolean;
+    conditionalImproved: boolean;
+    chineseMacroF1Improved: boolean;
+    englishMacroF1Improved: boolean;
+    outOfScopeImproved: boolean;
+    unavailableConservative: boolean;
+    observabilityComplete: boolean;
+}
+export declare function wilsonInterval95(successes: number, total: number): {
+    lower: number;
+    upper: number;
+};
+export declare function evaluateSemanticCalibration(samples: SemanticCalibrationSample[]): SemanticCalibrationReport;
+export declare function evaluateSemanticReleaseGates(input: {
+    report: SemanticQualityReport;
+    baseline?: SemanticQualityReport;
+    calibration?: SemanticCalibrationReport;
+    observabilityComplete: boolean;
+}): SemanticReleaseGates;
+export declare function evaluateSemanticFixtures(fixtures: SemanticIntentFixture[], predict: (prompt: string) => SemanticIntentExpectedFrame): SemanticQualityReport;
+//# sourceMappingURL=semantic-intent-quality.d.ts.map

package/dist/semantic-intent-quality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-intent-quality.d.ts","sourceRoot":"","sources":["../src/semantic-intent-quality.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,MAAM,MAAM,0BAA0B,GAClC,uBAAuB,GACvB,sBAAsB,GACtB,eAAe,GACf,aAAa,GACb,WAAW,CAAC;AAEhB,MAAM,WAAW,2BAA2B;IAC1C,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,mBAAmB,CAAC;IACnC,kBAAkB,EAAE,0BAA0B,CAAC;IAC/C,eAAe,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACrE,aAAa,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,GAAG,UAAU,CAAC;IACxE,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,GAAG,SAAS,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,2BAA2B,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,mCAAmC,CAAC;IACnD,QAAQ,EAAE,qBAAqB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,oCAAoC,CAAC;IACpD,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/E,mBAAmB,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,wBAAwB,EAAE,MAAM,CAAC;IACjC,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,sBAAsB,EAAE,OAAO,CAAC;IAChC,sBAAsB,EAAE,OAAO,CAAC;IAChC,kBAAkB,EAAE,OAAO,CAAC;IAC5B,uBAAuB,EAAE,OAAO,CAAC;IACjC,qBAAqB,EAAE,OAAO,CAAC;CAChC;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAQnG;AAED,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,yBAAyB,EAAE,GAAG,yBAAyB,CAyB3G;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE;IAClD,MAAM,EAAE,qBAAqB,CAAC;IAC9B,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,WAAW,CAAC,EAAE,yBAAyB,CAAC;IACxC,qBAAqB,EAAE,OAAO,CAAC;CAChC,GAAG,oBAAoB,CAavB;AAWD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,qBAAqB,EAAE,EACjC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,2BAA2B,GACvD,qBAAqB,CA0CvB"}

package/dist/semantic-intent-quality.js

@@ -0,0 +1,111 @@
+export function wilsonInterval95(successes, total) {
+    if (total <= 0)
+        return { lower: 0, upper: 0 };
+    const z = 1.959963984540054;
+    const p = successes / total;
+    const denominator = 1 + (z * z) / total;
+    const centre = (p + (z * z) / (2 * total)) / denominator;
+    const spread = z * Math.sqrt((p * (1 - p) / total) + (z * z) / (4 * total * total)) / denominator;
+    return { lower: Math.max(0, centre - spread), upper: Math.min(1, centre + spread) };
+}
+export function evaluateSemanticCalibration(samples) {
+    if (samples.length === 0)
+        return { brierScore: 0, expectedCalibrationError: 0, wilson95: { lower: 0, upper: 0 }, sampleCount: 0 };
+    const bins = Array.from({ length: 10 }, () => ({ count: 0, probability: 0, outcome: 0 }));
+    let brierScore = 0;
+    let successes = 0;
+    for (const sample of samples) {
+        const probability = Math.max(0, Math.min(1, sample.probability));
+        const outcome = sample.outcome ? 1 : 0;
+        brierScore += (probability - outcome) ** 2;
+        successes += outcome;
+        const bin = bins[Math.min(9, Math.floor(probability * 10))];
+        bin.count += 1;
+        bin.probability += probability;
+        bin.outcome += outcome;
+    }
+    const expectedCalibrationError = bins.reduce((sum, bin) => {
+        if (bin.count === 0)
+            return sum;
+        return sum + (bin.count / samples.length) * Math.abs(bin.probability / bin.count - bin.outcome / bin.count);
+    }, 0);
+    return {
+        brierScore: brierScore / samples.length,
+        expectedCalibrationError,
+        wilson95: wilsonInterval95(successes, samples.length),
+        sampleCount: samples.length,
+    };
+}
+export function evaluateSemanticReleaseGates(input) {
+    const baseline = input.baseline;
+    return {
+        mutationFalseAccept: input.report.mutationFalseAccept.numerator === 0,
+        structuralCompatibility: input.report.sampleCount > 0,
+        prohibitionImproved: !baseline || input.report.prohibitionAccuracy >= baseline.prohibitionAccuracy,
+        conditionalImproved: !baseline || input.report.macroF1 >= baseline.macroF1,
+        chineseMacroF1Improved: !baseline || input.report.macroF1 >= baseline.macroF1,
+        englishMacroF1Improved: !baseline || input.report.macroF1 >= baseline.macroF1,
+        outOfScopeImproved: !baseline || input.report.outOfScopeRecall >= baseline.outOfScopeRecall,
+        unavailableConservative: input.report.mutationFalseAccept.numerator === 0,
+        observabilityComplete: input.observabilityComplete && Boolean(input.calibration),
+    };
+}
+const FRAME_FIELDS = [
+    'primaryIntent',
+    'primaryAction',
+    'mutationPermission',
+    'memoryDirective',
+    'knowledgeKind',
+    'outOfScope',
+];
+export function evaluateSemanticFixtures(fixtures, predict) {
+    const diagnostics = [];
+    const predictions = fixtures.map(fixture => {
+        const actual = predict(fixture.prompt);
+        const fields = FRAME_FIELDS.filter(field => actual[field] !== fixture.expected[field]);
+        if (fields.length > 0)
+            diagnostics.push({ fixtureId: fixture.id, fields });
+        return { fixture, actual, fields };
+    });
+    const falseAcceptPopulation = predictions.filter(({ fixture }) => fixture.expected.mutationPermission !== 'explicitly_authorized');
+    const falseAcceptCount = falseAcceptPopulation.filter(({ actual }) => actual.mutationPermission === 'explicitly_authorized').length;
+    const falseRejectPopulation = predictions.filter(({ fixture }) => fixture.expected.mutationPermission === 'explicitly_authorized');
+    const falseRejectCount = falseRejectPopulation.filter(({ actual }) => actual.mutationPermission !== 'explicitly_authorized').length;
+    const prohibitions = predictions.filter(({ fixture }) => fixture.expected.mutationPermission === 'explicitly_forbidden');
+    const outOfScope = predictions.filter(({ fixture }) => fixture.expected.outOfScope);
+    return {
+        schemaVersion: 'devflow.semantic-quality-report.v1',
+        sampleCount: fixtures.length,
+        exactFrameMatch: ratio(predictions.filter(item => item.fields.length === 0).length, fixtures.length, diagnostics, 'exact-frame-empty'),
+        mutationFalseAccept: fraction(falseAcceptCount, falseAcceptPopulation.length, diagnostics, 'mutation-false-accept-empty'),
+        mutationFalseReject: fraction(falseRejectCount, falseRejectPopulation.length, diagnostics, 'mutation-false-reject-empty'),
+        prohibitionAccuracy: ratio(prohibitions.filter(({ actual }) => actual.mutationPermission === 'explicitly_forbidden').length, prohibitions.length, diagnostics, 'prohibition-empty'),
+        outOfScopeRecall: ratio(outOfScope.filter(({ actual }) => actual.outOfScope).length, outOfScope.length, diagnostics, 'out-of-scope-empty'),
+        macroF1: intentMacroF1(predictions),
+        diagnostics,
+    };
+}
+function fraction(numerator, denominator, diagnostics, diagnosticId) {
+    return { numerator, denominator, value: ratio(numerator, denominator, diagnostics, diagnosticId) };
+}
+function ratio(numerator, denominator, diagnostics, diagnosticId) {
+    if (denominator === 0) {
+        diagnostics.push({ fixtureId: diagnosticId, fields: ['denominator'] });
+        return 0;
+    }
+    return numerator / denominator;
+}
+function intentMacroF1(predictions) {
+    const labels = [...new Set(predictions.map(({ fixture }) => fixture.expected.primaryIntent))];
+    if (labels.length === 0)
+        return 0;
+    return labels.reduce((sum, label) => {
+        const truePositive = predictions.filter(({ fixture, actual }) => fixture.expected.primaryIntent === label && actual.primaryIntent === label).length;
+        const falsePositive = predictions.filter(({ fixture, actual }) => fixture.expected.primaryIntent !== label && actual.primaryIntent === label).length;
+        const falseNegative = predictions.filter(({ fixture, actual }) => fixture.expected.primaryIntent === label && actual.primaryIntent !== label).length;
+        const precision = truePositive + falsePositive === 0 ? 0 : truePositive / (truePositive + falsePositive);
+        const recall = truePositive + falseNegative === 0 ? 0 : truePositive / (truePositive + falseNegative);
+        return sum + (precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall));
+    }, 0) / labels.length;
+}
+//# sourceMappingURL=semantic-intent-quality.js.map

package/dist/semantic-intent-quality.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-intent-quality.js","sourceRoot":"","sources":["../src/semantic-intent-quality.ts"],"names":[],"mappings":"AAqEA,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,KAAa;IAC/D,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC9C,MAAM,CAAC,GAAG,iBAAiB,CAAC;IAC5B,MAAM,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;IAC5B,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC;IAClG,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AACtF,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAoC;IAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,wBAAwB,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IAClI,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1F,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,UAAU,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,SAAS,IAAI,OAAO,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;QAC7D,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;QAAC,GAAG,CAAC,WAAW,IAAI,WAAW,CAAC;QAAC,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC;IACzE,CAAC;IACD,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACxD,IAAI,GAAG,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAChC,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAClD,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CACtD,CAAC;IACJ,CAAC,EAAE,CAAC,CAAC,CAAC;IACN,OAAO;QACL,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM;QACvC,wBAAwB;QACxB,QAAQ,EAAE,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC;QACrD,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAK5C;IACC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAChC,OAAO;QACL,mBAAmB,EAAE,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,SAAS,KAAK,CAAC;QACrE,uBAAuB,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC;QACrD,mBAAmB,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,mBAAmB,IAAI,QAAQ,CAAC,mBAAmB;QAClG,mBAAmB,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;QAC1E,sBAAsB,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;QAC7E,sBAAsB,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;QAC7E,kBAAkB,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB;QAC3F,uBAAuB,EAAE,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,SAAS,KAAK,CAAC;QACzE,qBAAqB,EAAE,KAAK,CAAC,qBAAqB,IAAI,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;KACjF,CAAC;AACJ,CAAC;AAED,MAAM,YAAY,GAA6C;IAC7D,eAAe;IACf,eAAe;IACf,oBAAoB;IACpB,iBAAiB;IACjB,eAAe;IACf,YAAY;CACb,CAAC;AAEF,MAAM,UAAU,wBAAwB,CACtC,QAAiC,EACjC,OAAwD;IAExD,MAAM,WAAW,GAAyC,EAAE,CAAC;IAC7D,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;QACzC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,MAAM,qBAAqB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAC/D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,KAAK,uBAAuB,CAAC,CAAC;IACnE,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACnE,MAAM,CAAC,kBAAkB,KAAK,uBAAuB,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,qBAAqB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAC/D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,KAAK,uBAAuB,CAAC,CAAC;IACnE,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACnE,MAAM,CAAC,kBAAkB,KAAK,uBAAuB,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CACtD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,KAAK,sBAAsB,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAEpF,OAAO;QACL,aAAa,EAAE,oCAAoC;QACnD,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,eAAe,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,mBAAmB,CAAC;QACtI,mBAAmB,EAAE,QAAQ,CAAC,gBAAgB,EAAE,qBAAqB,CAAC,MAAM,EAAE,WAAW,EAAE,6BAA6B,CAAC;QACzH,mBAAmB,EAAE,QAAQ,CAAC,gBAAgB,EAAE,qBAAqB,CAAC,MAAM,EAAE,WAAW,EAAE,6BAA6B,CAAC;QACzH,mBAAmB,EAAE,KAAK,CACxB,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,kBAAkB,KAAK,sBAAsB,CAAC,CAAC,MAAM,EAChG,YAAY,CAAC,MAAM,EACnB,WAAW,EACX,mBAAmB,CACpB;QACD,gBAAgB,EAAE,KAAK,CACrB,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAC3D,UAAU,CAAC,MAAM,EACjB,WAAW,EACX,oBAAoB,CACrB;QACD,OAAO,EAAE,aAAa,CAAC,WAAW,CAAC;QACnC,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,SAAiB,EACjB,WAAmB,EACnB,WAAiD,EACjD,YAAoB;IAEpB,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,CAAC;AACrG,CAAC;AAED,SAAS,KAAK,CACZ,SAAiB,EACjB,WAAmB,EACnB,WAAiD,EACjD,YAAoB;IAEpB,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,SAAS,GAAG,WAAW,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CACpB,WAA2F;IAE3F,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC9F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QAClC,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAC9D,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,KAAK,IAAI,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrF,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/D,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,KAAK,IAAI,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrF,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAC/D,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,KAAK,IAAI,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrF,MAAM,SAAS,GAAG,YAAY,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC;QACzG,MAAM,MAAM,GAAG,YAAY,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC;QACtG,OAAO,GAAG,GAAG,CAAC,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IAChG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACxB,CAAC"}

package/fixtures/semantic-intent-corpus.json

@@ -0,0 +1,125 @@
+{
+  "schemaVersion": "devflow.semantic-intent-corpus.v1",
+  "fixtures": [
+    { "id": "debug-mutate-01", "family": "debug_mutate", "partition": "development", "prompt": "fix the login page crash", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-02", "family": "debug_mutate", "partition": "development", "prompt": "修复登录崩溃", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-03", "family": "debug_mutate", "partition": "development", "prompt": "debug the failing token refresh", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-04", "family": "debug_mutate", "partition": "development", "prompt": "解决支付回调异常", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-05", "family": "debug_mutate", "partition": "development", "prompt": "fix the useAuth hook in LoginPage", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-06", "family": "debug_mutate", "partition": "development", "prompt": "请直接修复 build failure", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+
+    { "id": "feature-mutate-01", "family": "feature_mutate", "partition": "development", "prompt": "add a new search feature", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-02", "family": "feature_mutate", "partition": "development", "prompt": "帮我生成一个登录组件", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-03", "family": "feature_mutate", "partition": "development", "prompt": "create a Login component", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-04", "family": "feature_mutate", "partition": "development", "prompt": "build a Login component", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-05", "family": "feature_mutate", "partition": "development", "prompt": "编写登录组件", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-06", "family": "feature_mutate", "partition": "development", "prompt": "创建组件 Login", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "analysis-only-01", "family": "analysis_only", "partition": "development", "prompt": "how does the auth system work", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-02", "family": "analysis_only", "partition": "development", "prompt": "这个实现没有 bug，只想了解它为什么这样设计", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-03", "family": "analysis_only", "partition": "development", "prompt": "analyze the current VideoPlayer implementation in src/components/video-player/VideoPlayer.tsx", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-04", "family": "analysis_only", "partition": "development", "prompt": "分析当前登录模块的实现逻辑", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-05", "family": "analysis_only", "partition": "development", "prompt": "how to implement a cache without changing the current code", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-06", "family": "analysis_only", "partition": "development", "prompt": "查看现有组件实现方式", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-07", "family": "analysis_only", "partition": "development", "prompt": "review the existing authentication implementation", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+
+    { "id": "release-mutate-01", "family": "release_mutate", "partition": "development", "prompt": "publish version 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-02", "family": "release_mutate", "partition": "development", "prompt": "发布 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-03", "family": "release_mutate", "partition": "development", "prompt": "release the approved package", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-04", "family": "release_mutate", "partition": "development", "prompt": "开始发版 npm 包", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-05", "family": "release_mutate", "partition": "development", "prompt": "publish the current workspace as 0.18.1", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-06", "family": "release_mutate", "partition": "development", "prompt": "请发布新版本", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "memory-save-01", "family": "memory_save", "partition": "development", "prompt": "remember that this project uses pnpm", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-02", "family": "memory_save", "partition": "development", "prompt": "记住项目统一使用 pnpm", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-03", "family": "memory_save", "partition": "development", "prompt": "save this convention to project memory", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-04", "family": "memory_save", "partition": "development", "prompt": "请记住我们不使用 yarn", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-05", "family": "memory_save", "partition": "development", "prompt": "请记住项目使用 TanStack Router，不修改项目文件", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-06", "family": "memory_save", "partition": "development", "prompt": "store the API naming rule for later", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+
+    { "id": "knowledge-api-01", "family": "knowledge_api", "partition": "development", "prompt": "query the React 19 use API", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-02", "family": "knowledge_api", "partition": "development", "prompt": "查询 React 19 use API", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-03", "family": "knowledge_api", "partition": "development", "prompt": "what is the signature of useActionState", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-04", "family": "knowledge_api", "partition": "development", "prompt": "查一下 BGE-M3 的调用参数", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-05", "family": "knowledge_api", "partition": "development", "prompt": "show the MCP sampling method reference", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-06", "family": "knowledge_api", "partition": "development", "prompt": "解释 React hooks 的原理", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "overview", "outOfScope": false } },
+
+    { "id": "conditional-mutation-01", "family": "conditional_mutation", "partition": "development", "prompt": "analyze the issue and implement the approved solution", "expected": { "primaryIntent": "debug", "primaryAction": "create", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-02", "family": "conditional_mutation", "partition": "development", "prompt": "先分析登录崩溃，确认根因后再修复", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-03", "family": "conditional_mutation", "partition": "development", "prompt": "if the tests reproduce it, fix the cache race", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-04", "family": "conditional_mutation", "partition": "development", "prompt": "如果能确认类型错误，再修改接口", "expected": { "primaryIntent": "debug", "primaryAction": "modify", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-05", "family": "conditional_mutation", "partition": "development", "prompt": "分析完成后实现这个方案", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-mutation-06", "family": "conditional_mutation", "partition": "development", "prompt": "once I approve the diff, apply the refactor", "expected": { "primaryIntent": "refactor", "primaryAction": "modify", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "negated-debug-01", "family": "negated_debug", "partition": "heldout", "prompt": "The build command is failing; explain why, do not execute anything", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-02", "family": "negated_debug", "partition": "heldout", "prompt": "登录一直崩溃，只分析原因，不要修复", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-03", "family": "negated_debug", "partition": "heldout", "prompt": "diagnose the failing test without changing files", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-04", "family": "negated_debug", "partition": "heldout", "prompt": "解释缓存 race，不执行命令也不改代码", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-05", "family": "negated_debug", "partition": "heldout", "prompt": "review the crash report; never run the fix", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-06", "family": "negated_debug", "partition": "heldout", "prompt": "分析构建失败，但不要重新构建", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+
+    { "id": "negated-release-analysis", "family": "negated_release", "partition": "heldout", "prompt": "如果发布失败，只分析日志，不要重新发布", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-release-02", "family": "negated_release", "partition": "heldout", "prompt": "analyze why 0.17.3 publish failed; do not republish", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-release-03", "family": "negated_release", "partition": "heldout", "prompt": "查看发版错误，不要运行 npm publish", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-release-04", "family": "negated_release", "partition": "heldout", "prompt": "explain the release pipeline without publishing", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negated-release-05", "family": "negated_release", "partition": "heldout", "prompt": "审查发布脚本，禁止发版", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negated-release-06", "family": "negated_release", "partition": "heldout", "prompt": "find the npm publish failure but make no release", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+
+    { "id": "memory-forget-recall-01", "family": "memory_forget_recall", "partition": "heldout", "prompt": "删除“旧缓存策略”这条记忆", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "forget", "outOfScope": false } },
+    { "id": "memory-forget-recall-02", "family": "memory_forget_recall", "partition": "heldout", "prompt": "forget the old cache policy", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "forget", "outOfScope": false } },
+    { "id": "memory-forget-recall-03", "family": "memory_forget_recall", "partition": "heldout", "prompt": "回忆项目选择路由库的原因", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "recall", "outOfScope": false } },
+    { "id": "memory-forget-recall-04", "family": "memory_forget_recall", "partition": "heldout", "prompt": "recall why we chose the router", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "recall", "outOfScope": false } },
+    { "id": "memory-forget-recall-05", "family": "memory_forget_recall", "partition": "heldout", "prompt": "别记住这个临时 pnpm 命令", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "memory-forget-recall-06", "family": "memory_forget_recall", "partition": "heldout", "prompt": "记一下还是算了", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "ambiguous", "outOfScope": false } },
+
+    { "id": "knowledge-migration-01", "family": "knowledge_migration", "partition": "heldout", "prompt": "migrate React 18 to React 19", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-02", "family": "knowledge_migration", "partition": "heldout", "prompt": "迁移 React 18 到 React 19", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-03", "family": "knowledge_migration", "partition": "heldout", "prompt": "what are the breaking changes in Vue 4", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-04", "family": "knowledge_migration", "partition": "heldout", "prompt": "查询 Node 24 升级指南", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-05", "family": "knowledge_migration", "partition": "heldout", "prompt": "compare the migration notes for Vite 7", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-06", "family": "knowledge_migration", "partition": "heldout", "prompt": "了解 TypeScript 6 迁移风险", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+
+    { "id": "negative-performance-01", "family": "negative_performance", "partition": "heldout", "prompt": "not a performance problem; review naming only", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "negative-performance-02", "family": "negative_performance", "partition": "heldout", "prompt": "不是性能问题，只分析变量命名", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "negative-performance-03", "family": "negative_performance", "partition": "heldout", "prompt": "分析 React 组件渲染性能", "expected": { "primaryIntent": "performance", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negative-performance-04", "family": "negative_performance", "partition": "heldout", "prompt": "不要优化性能，只解释渲染顺序", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negative-performance-05", "family": "negative_performance", "partition": "heldout", "prompt": "this is not a memory leak; explain ownership", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negative-performance-06", "family": "negative_performance", "partition": "heldout", "prompt": "不处理卡顿，审查组件职责即可", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "oos-01", "family": "out_of_scope", "partition": "heldout", "prompt": "天气怎么样", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-02", "family": "out_of_scope", "partition": "heldout", "prompt": "write a poem about the ocean", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-03", "family": "out_of_scope", "partition": "heldout", "prompt": "上海明天限行吗", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-04", "family": "out_of_scope", "partition": "heldout", "prompt": "recommend a dinner recipe", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-05", "family": "out_of_scope", "partition": "heldout", "prompt": "讲一个冷笑话", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-06", "family": "out_of_scope", "partition": "heldout", "prompt": "translate this legal contract", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-07", "family": "out_of_scope", "partition": "heldout", "prompt": "hmm", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+
+    { "id": "review-nonmutating-01", "family": "review_nonmutating", "partition": "heldout", "prompt": "review LoginPage without modifying the project", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-02", "family": "review_nonmutating", "partition": "heldout", "prompt": "审查 LoginPage，不要创建或修改任何文件", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-03", "family": "review_nonmutating", "partition": "heldout", "prompt": "please critique the component; no file changes", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-04", "family": "review_nonmutating", "partition": "heldout", "prompt": "只做代码审查，禁止编辑", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-05", "family": "review_nonmutating", "partition": "heldout", "prompt": "assess the architecture and do not implement recommendations", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-06", "family": "review_nonmutating", "partition": "heldout", "prompt": "检查类型设计，不要提交修改", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "conditional-release-01", "family": "conditional_release", "partition": "heldout", "prompt": "if all tests pass, publish 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-02", "family": "conditional_release", "partition": "heldout", "prompt": "测试全部通过后再发布 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-03", "family": "conditional_release", "partition": "heldout", "prompt": "publish only after I approve the changelog", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-04", "family": "conditional_release", "partition": "heldout", "prompt": "等我确认版本号之后发版", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-05", "family": "conditional_release", "partition": "heldout", "prompt": "when CI is green, run npm publish", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-06", "family": "conditional_release", "partition": "heldout", "prompt": "如果 tag 尚未存在，就发布新版本", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "ambiguous-followup-01", "family": "ambiguous_followup", "partition": "heldout", "prompt": "继续", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-02", "family": "ambiguous_followup", "partition": "heldout", "prompt": "按刚才批准的方案执行", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-03", "family": "ambiguous_followup", "partition": "heldout", "prompt": "do it", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-04", "family": "ambiguous_followup", "partition": "heldout", "prompt": "用上一个", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-05", "family": "ambiguous_followup", "partition": "heldout", "prompt": "apply that change", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-06", "family": "ambiguous_followup", "partition": "heldout", "prompt": "就这么办", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+
+    { "id": "explicit-implementation-01", "family": "explicit_implementation", "partition": "development", "prompt": "implement a Login component", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-02", "family": "explicit_implementation", "partition": "development", "prompt": "implement a cache for the authentication module", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-03", "family": "explicit_implementation", "partition": "development", "prompt": "帮我实现登录组件", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-04", "family": "explicit_implementation", "partition": "development", "prompt": "新增缓存模块", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-05", "family": "explicit_implementation", "partition": "development", "prompt": "scaffold a settings page", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-06", "family": "explicit_implementation", "partition": "development", "prompt": "开发一个用户搜索功能", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } }
+  ]
+}

package/fixtures/semantic-regex-inventory.json

@@ -0,0 +1,20 @@
+{
+  "schemaVersion": "devflow.semantic-regex-inventory.v1",
+  "entries": [
+    { "file": "packages/context-engine/src/intent-parser.ts", "symbol": "INTENT_PATTERNS", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryIntent" },
+    { "file": "packages/context-engine/src/intent-parser.ts", "symbol": "CN_INTENT_MAP", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryIntent" },
+    { "file": "packages/context-engine/src/intent-parser.ts", "symbol": "ZH_EN_MAP", "responsibility": "lexical_feature", "disposition": "retain_for_retrieval_only" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "canonicalIntent", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryIntent" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "inferAction", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryAction" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "extractKnowledgeSymbols", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "extractEntityVersions", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/memory-engine/src/signal-analyzer.ts", "symbol": "detectCheap", "scope": "user_message", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.memoryDirective" },
+    { "file": "packages/memory-engine/src/signal-analyzer.ts", "symbol": "detectCheap", "scope": "tool_event", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/memory-engine/src/explicit-memory-enricher.ts", "symbol": "classifyExplicitMemory", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.memoryDirective" },
+    { "file": "packages/memory-engine/src/observation-quality.ts", "symbol": "classifyFailureCategory", "responsibility": "lexical_feature", "disposition": "retain_non_authoritative" },
+    { "file": "packages/knowledge-engine/src/source-selector.ts", "symbol": "SOURCE_FAMILIES", "scope": "query", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.knowledgeNeeds" },
+    { "file": "packages/knowledge-engine/src/source-selector.ts", "symbol": "SOURCE_FAMILIES", "scope": "source/dependencies/frameworks/languages/files", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/devflow/src/hooks/evidence-contract.ts", "symbol": "validateEvidenceContract", "responsibility": "safety_defense", "disposition": "keep" },
+    { "file": "packages/devflow/src/hooks/slash-command.ts", "symbol": "parseSlashCommand", "responsibility": "structural_authority", "disposition": "keep" }
+  ]
+}

package/fixtures/semantic-thresholds.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": "semantic-thresholds.v1",
+  "version": "semantic-intent-corpus.v1",
+  "modelId": "Xenova/all-MiniLM-L6-v2",
+  "modelRevision": "751bff37182d3f1213fa05d7196b954e230abad9",
+  "routeCatalogVersion": "semantic-route-catalog.v1",
+  "minimumScore": 0.55,
+  "minimumMargin": 0.08,
+  "fitPartition": "development"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devflow-tools/benchmark",
-  "version": "0.17.3",
+  "version": "0.17.4",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -19,9 +19,9 @@
   },
   "dependencies": {
     "@colbymchenry/codegraph": "^1.1.1",
-    "@devflow-tools/context-engine": "0.17.3",
-    "@devflow-tools/database": "0.17.3",
-    "@devflow-tools/sdk": "0.17.3"
+    "@devflow-tools/context-engine": "0.17.4",
+    "@devflow-tools/database": "0.17.4",
+    "@devflow-tools/sdk": "0.17.4"
   },
   "devDependencies": {
     "typescript": "^5.5.0",
@@ -31,5 +31,5 @@
     "dist",
     "fixtures"
   ],
-  "gitHead": "964af33767bf232606e63d973b3a29d7d53c8119"
+  "gitHead": "b880054f9cef4ed62b4d02544257c9f5cbd811bc"
 }