@alan512/experienceengine 0.1.3 → 0.2.1

package/dist/evaluation/codex-lifecycle-validation.d.ts

@@ -0,0 +1,60 @@
+export type CodexLifecycleValidationOptions = {
+    env?: NodeJS.ProcessEnv;
+    homeDir?: string;
+    outputDir?: string;
+    repoRoot?: string;
+    now?: () => string;
+};
+export type CodexLifecycleValidationReport = {
+    generatedAt: string;
+    repoRoot: string;
+    prompt: string;
+    sessionId: string;
+    outputDir: string;
+    runtimeHome: string;
+    sqlitePath: string;
+    captureDir: string;
+    seededNodeId: string;
+    lookup: {
+        mode: "skip" | "inject_conservative" | "inject";
+        injectedNodeIds: string[];
+        deliveryMode?: "live" | "shadow" | "holdout";
+        delivered?: boolean;
+        notice?: string;
+    };
+    toolResult: {
+        status: string;
+        eventStatus: string;
+        toolName: string;
+    };
+    finalize: {
+        status: string;
+        outcomeSignal: string;
+        recordedToolEvents: number;
+    };
+    persistence: {
+        taskRunCount: number;
+        injectionEventCount: number;
+        reviewEventCount: number;
+        hybridArtifactCount: number;
+        hybridTraceCount: number;
+        reviewEventTypes: string[];
+    };
+    node: {
+        id: string;
+        state: string;
+        deliveryState: string;
+        usageCount: number;
+        helpedCount: number;
+        harmedCount: number;
+        lastFeedbackVerdict?: string;
+    };
+};
+export type CodexLifecycleValidationRunResult = {
+    outputDir: string;
+    jsonPath: string;
+    markdownPath: string;
+    report: CodexLifecycleValidationReport;
+};
+export declare const renderCodexLifecycleValidationMarkdown: (report: CodexLifecycleValidationReport) => string;
+export declare const runCodexLifecycleValidation: (options?: CodexLifecycleValidationOptions) => Promise<CodexLifecycleValidationRunResult>;

package/dist/evaluation/codex-lifecycle-validation.js

@@ -0,0 +1,233 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { loadConfig } from "../config/load-config.js";
+import { createCodexBehaviorLoop } from "../adapters/codex/mcp-server.js";
+import { resolveScope } from "../input/scope-resolver.js";
+import { openDatabase, bootstrapDatabase } from "../store/sqlite/db.js";
+import { HybridInvocationTraceRepository } from "../store/sqlite/repositories/hybrid-invocation-trace-repo.js";
+import { HybridReviewArtifactRepository } from "../store/sqlite/repositories/hybrid-review-artifact-repo.js";
+import { InjectionRepository } from "../store/sqlite/repositories/injection-repo.js";
+import { NodeRepository } from "../store/sqlite/repositories/node-repo.js";
+import { ReviewEventRepository } from "../store/sqlite/repositories/review-event-repo.js";
+import { TaskRunRepository } from "../store/sqlite/repositories/task-run-repo.js";
+import { buildLegacyEmbedding } from "../store/vector/embeddings.js";
+const DEFAULT_SESSION_ID = "codex-lifecycle-validation";
+const DEFAULT_NODE_ID = "node_codex_lifecycle_validation";
+const DEFAULT_PROMPT = "Fix the failing auth test";
+const sanitizeStamp = (value) => value.replace(/[:.]/g, "-");
+const mkdirIfMissing = (path) => {
+    if (!existsSync(path)) {
+        mkdirSync(path, { recursive: true });
+    }
+};
+const defaultOutputDir = (timestamp) => resolve("artifacts", "evaluations", "codex", sanitizeStamp(timestamp));
+const seedLifecycleNode = (repo, repoRoot, timestamp, nodeId, prompt) => {
+    const scope = resolveScope(repoRoot);
+    const retrievalText = `${prompt}\nRun the failing auth test before editing and verify after the fix.`;
+    const embedding = buildLegacyEmbedding(retrievalText);
+    const node = {
+        id: nodeId,
+        node_type: "strategy",
+        scope_id: scope.scope_id,
+        task_type: "test_debug",
+        trigger_pattern: prompt,
+        applicability_notes: "Use the same repo and test scope",
+        env_signature: undefined,
+        compact_hint: "Run the failing auth test before editing and verify after the fix.",
+        goal: "Stabilize the failing auth test",
+        recommended_steps: ["Run the failing test", "Apply the minimal fix", "Re-run the test"],
+        avoid_steps: [],
+        fallback_steps: [],
+        success_signal: "The targeted auth test passes",
+        stop_condition: undefined,
+        escalation_condition: undefined,
+        evidence_summary: "Recovered the same failing auth test in a prior Codex lifecycle validation run.",
+        retrieval_text: retrievalText,
+        embedding: embedding.embedding,
+        embedding_provider: embedding.space.provider,
+        embedding_model: embedding.space.model,
+        embedding_version: embedding.space.version,
+        embedding_dimensions: embedding.space.dimensions,
+        distillation_mode_used: "rule",
+        distillation_source: "rule",
+        source_kind: "system_derived",
+        origin_record_ids: ["input_codex_validation_origin"],
+        helped_record_ids: [],
+        harmed_record_ids: [],
+        state: "active",
+        delivery_state: "eligible",
+        usage_count: 0,
+        helped_count: 0,
+        harmed_count: 0,
+        consecutive_harmed_count: 0,
+        support_count: 1,
+        created_at: timestamp,
+        updated_at: timestamp
+    };
+    repo.upsert(node);
+};
+export const renderCodexLifecycleValidationMarkdown = (report) => [
+    "# Codex lifecycle validation",
+    "",
+    `- Generated at: ${report.generatedAt}`,
+    `- Repo root: ${report.repoRoot}`,
+    `- Session id: ${report.sessionId}`,
+    `- Seeded node: ${report.seededNodeId}`,
+    `- Lookup mode: ${report.lookup.mode}`,
+    `- Injected node ids: ${report.lookup.injectedNodeIds.join(", ") || "none"}`,
+    `- Final outcome: ${report.finalize.outcomeSignal}`,
+    `- Persisted task runs: ${report.persistence.taskRunCount}`,
+    `- Persisted injection events: ${report.persistence.injectionEventCount}`,
+    `- Persisted review events: ${report.persistence.reviewEventCount}`,
+    `- Persisted hybrid artifacts: ${report.persistence.hybridArtifactCount}`,
+    `- Persisted hybrid traces: ${report.persistence.hybridTraceCount}`,
+    `- Review event types: ${report.persistence.reviewEventTypes.join(", ") || "none"}`,
+    `- Node lifecycle: ${report.node.state}`,
+    `- Node delivery: ${report.node.deliveryState}`,
+    `- Node helped/harmed: ${report.node.helpedCount}/${report.node.harmedCount}`
+].join("\n");
+export const runCodexLifecycleValidation = async (options = {}) => {
+    const generatedAt = options.now?.() ?? new Date().toISOString();
+    const outputDir = resolve(options.outputDir ?? defaultOutputDir(generatedAt));
+    const runtimeHome = resolve(options.homeDir ?? join(outputDir, "runtime-home"));
+    const experienceHome = join(runtimeHome, ".experienceengine");
+    const repoRoot = options.repoRoot ? resolve(options.repoRoot) : process.cwd();
+    const prompt = DEFAULT_PROMPT;
+    mkdirIfMissing(outputDir);
+    mkdirIfMissing(runtimeHome);
+    const env = {
+        ...process.env,
+        ...options.env,
+        EXPERIENCE_ENGINE_HOME: experienceHome,
+        EXPERIENCE_ENGINE_EMBEDDING_PROVIDER: "legacy",
+        EXPERIENCE_ENGINE_DISTILLATION_MODE: "disabled",
+        EXPERIENCE_ENGINE_HYBRID_ENABLED: "true",
+        EXPERIENCE_ENGINE_HYBRID_ASYNC_POSTMORTEM_ENABLED: "true",
+        EXPERIENCE_ENGINE_HYBRID_ASYNC_POSTMORTEM_LLM_ENABLED: "false",
+        EXPERIENCE_ENGINE_HYBRID_ROLLOUT_MODE: "live"
+    };
+    const config = loadConfig({}, { env, homeDir: runtimeHome });
+    const db = openDatabase(config);
+    bootstrapDatabase(db);
+    const nodeRepo = new NodeRepository(db);
+    seedLifecycleNode(nodeRepo, repoRoot, generatedAt, DEFAULT_NODE_ID, prompt);
+    const loop = createCodexBehaviorLoop({
+        env,
+        homeDir: runtimeHome,
+        runtimeOptions: {
+            env,
+            homeDir: runtimeHome,
+            hybridWorkerClientOptions: {
+                postmortemReviewExecutor: async () => ({
+                    task: "postmortem_review",
+                    review_verdict: "policy_gated",
+                    candidate_recommendation: "observe",
+                    feedback_followup_recommendation: "none",
+                    confidence: "high",
+                    reason: "The injected node materially contributed to the successful Codex lifecycle validation run.",
+                    review_artifact: {
+                        summary: "The injected node materially contributed to the successful Codex lifecycle validation run.",
+                        notes: ["Apply bounded helped writeback for the injected node."]
+                    },
+                    injected_node_reviews: [
+                        {
+                            node_id: DEFAULT_NODE_ID,
+                            feedback_verdict: "helped",
+                            confidence: "high",
+                            delivery_recommendation: "keep",
+                            reason: "The deterministic validation flow followed the injected verification loop and completed successfully."
+                        }
+                    ]
+                })
+            }
+        }
+    });
+    const lookup = await loop.lookupHints({
+        cwd: repoRoot,
+        prompt,
+        sessionId: DEFAULT_SESSION_ID
+    });
+    const toolResult = await loop.recordToolResult({
+        sessionId: DEFAULT_SESSION_ID,
+        toolName: "vitest",
+        inputSummary: "pnpm vitest run auth",
+        outputSummary: "The targeted auth test passed after following the injected hint.",
+        status: "success"
+    });
+    const finalized = await loop.finalizeTask({
+        sessionId: DEFAULT_SESSION_ID,
+        cwd: repoRoot,
+        prompt
+    });
+    await loop.waitForBackgroundLearning();
+    const taskRunRepo = new TaskRunRepository(db);
+    const injectionRepo = new InjectionRepository(db);
+    const reviewRepo = new ReviewEventRepository(db);
+    const artifactRepo = new HybridReviewArtifactRepository(db);
+    const traceRepo = new HybridInvocationTraceRepository(db);
+    const taskRun = taskRunRepo.getLatestBySessionId(DEFAULT_SESSION_ID);
+    if (!taskRun) {
+        throw new Error("Codex lifecycle validation expected a persisted task run.");
+    }
+    const node = nodeRepo.getById(DEFAULT_NODE_ID);
+    if (!node) {
+        throw new Error("Codex lifecycle validation expected the seeded node to remain persisted.");
+    }
+    const reviewEvents = reviewRepo.listByTaskRunId(taskRun.id).reverse();
+    const report = {
+        generatedAt,
+        repoRoot,
+        prompt,
+        sessionId: DEFAULT_SESSION_ID,
+        outputDir,
+        runtimeHome,
+        sqlitePath: config.sqlitePath,
+        captureDir: config.captureDir,
+        seededNodeId: DEFAULT_NODE_ID,
+        lookup: {
+            mode: lookup.mode,
+            injectedNodeIds: lookup.injectedNodeIds,
+            deliveryMode: lookup.deliveryMode,
+            delivered: lookup.delivered,
+            notice: lookup.notice
+        },
+        toolResult: {
+            status: toolResult.status,
+            eventStatus: toolResult.eventStatus,
+            toolName: toolResult.toolName
+        },
+        finalize: {
+            status: finalized.status,
+            outcomeSignal: finalized.outcomeSignal,
+            recordedToolEvents: finalized.recordedToolEvents
+        },
+        persistence: {
+            taskRunCount: taskRunRepo.count(),
+            injectionEventCount: injectionRepo.count(),
+            reviewEventCount: reviewRepo.count(),
+            hybridArtifactCount: artifactRepo.count(),
+            hybridTraceCount: traceRepo.count(),
+            reviewEventTypes: reviewEvents.map((event) => event.event_type)
+        },
+        node: {
+            id: node.id,
+            state: node.state,
+            deliveryState: node.delivery_state ?? "eligible",
+            usageCount: node.usage_count,
+            helpedCount: node.helped_count,
+            harmedCount: node.harmed_count,
+            lastFeedbackVerdict: node.last_feedback_verdict
+        }
+    };
+    const jsonPath = join(outputDir, "codex-lifecycle.json");
+    const markdownPath = join(outputDir, "codex-lifecycle.md");
+    writeFileSync(jsonPath, JSON.stringify(report, null, 2));
+    writeFileSync(markdownPath, renderCodexLifecycleValidationMarkdown(report));
+    return {
+        outputDir,
+        jsonPath,
+        markdownPath,
+        report
+    };
+};
+//# sourceMappingURL=codex-lifecycle-validation.js.map

package/dist/evaluation/codex-lifecycle-validation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-lifecycle-validation.js","sourceRoot":"","sources":["../../src/evaluation/codex-lifecycle-validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,+BAA+B,EAAE,MAAM,8DAA8D,CAAC;AAC/G,OAAO,EAAE,8BAA8B,EAAE,MAAM,6DAA6D,CAAC;AAC7G,OAAO,EAAE,mBAAmB,EAAE,MAAM,gDAAgD,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAC1F,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAgErE,MAAM,kBAAkB,GAAG,4BAA4B,CAAC;AACxD,MAAM,eAAe,GAAG,iCAAiC,CAAC;AAC1D,MAAM,cAAc,GAAG,2BAA2B,CAAC;AAEnD,MAAM,aAAa,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAE7E,MAAM,cAAc,GAAG,CAAC,IAAY,EAAQ,EAAE;IAC5C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,SAAiB,EAAU,EAAE,CACrD,OAAO,CAAC,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC;AAEzE,MAAM,iBAAiB,GAAG,CACxB,IAAoB,EACpB,QAAgB,EAChB,SAAiB,EACjB,MAAc,EACd,MAAc,EACR,EAAE;IACR,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,aAAa,GAAG,GAAG,MAAM,sEAAsE,CAAC;IACtG,MAAM,SAAS,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,IAAI,GAAmB;QAC3B,EAAE,EAAE,MAAM;QACV,SAAS,EAAE,UAAU;QACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,YAAY;QACvB,eAAe,EAAE,MAAM;QACvB,mBAAmB,EAAE,kCAAkC;QACvD,aAAa,EAAE,SAAS;QACxB,YAAY,EAAE,oEAAoE;QAClF,IAAI,EAAE,iCAAiC;QACvC,iBAAiB,EAAE,CAAC,sBAAsB,EAAE,uBAAuB,EAAE,iBAAiB,CAAC;QACvF,WAAW,EAAE,EAAE;QACf,cAAc,EAAE,EAAE;QAClB,cAAc,EAAE,+BAA+B;QAC/C,cAAc,EAAE,SAAS;QACzB,oBAAoB,EAAE,SAAS;QAC/B,gBAAgB,EAAE,iFAAiF;QACnG,cAAc,EAAE,aAAa;QAC7B,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;QAC5C,eAAe,EAAE,SAAS,CAAC,KAAK,CAAC,KAAK;QACtC,iBAAiB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;QAC1C,oBAAoB,EAAE,SAAS,CAAC,KAAK,CAAC,UAAU;QAChD,sBAAsB,EAAE,MAAM;QAC9B,mBAAmB,EAAE,MAAM;QAC3B,WAAW,EAAE,gBAAgB;QAC7B,iBAAiB,EAAE,CAAC,+BAA+B,CAAC;QACpD,iBAAiB,EAAE,EAAE;QACrB,iBAAiB,EAAE,EAAE;QACrB,KAAK,EAAE,QAAQ;QACf,cAAc,EAAE,UAAU;QAC1B,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,wBAAwB,EAAE,CAAC;QAC3B,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS;KACtB,CAAC;IAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sCAAsC,GAAG,CACpD,MAAsC,EAC9B,EAAE,CACV;IACE,8BAA8B;IAC9B,EAAE;IACF,mBAAmB,MAAM,CAAC,WAAW,EAAE;IACvC,gBAAgB,MAAM,CAAC,QAAQ,EAAE;IACjC,iBAAiB,MAAM,CAAC,SAAS,EAAE;IACnC,kBAAkB,MAAM,CAAC,YAAY,EAAE;IACvC,kBAAkB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;IACtC,wBAAwB,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;IAC5E,oBAAoB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE;IACnD,0BAA0B,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE;IAC3D,iCAAiC,MAAM,CAAC,WAAW,CAAC,mBAAmB,EAAE;IACzE,8BAA8B,MAAM,CAAC,WAAW,CAAC,gBAAgB,EAAE;IACnE,iCAAiC,MAAM,CAAC,WAAW,CAAC,mBAAmB,EAAE;IACzE,8BAA8B,MAAM,CAAC,WAAW,CAAC,gBAAgB,EAAE;IACnE,yBAAyB,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;IACnF,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE;IACxC,oBAAoB,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE;IAC/C,yBAAyB,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;CAC9E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEf,MAAM,CAAC,MAAM,2BAA2B,GAAG,KAAK,EAC9C,UAA2C,EAAE,EACD,EAAE;IAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;IAC9E,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;IAC9E,MAAM,MAAM,GAAG,cAAc,CAAC;IAE9B,cAAc,CAAC,SAAS,CAAC,CAAC;IAC1B,cAAc,CAAC,WAAW,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAsB;QAC7B,GAAG,OAAO,CAAC,GAAG;QACd,GAAG,OAAO,CAAC,GAAG;QACd,sBAAsB,EAAE,cAAc;QACtC,oCAAoC,EAAE,QAAQ;QAC9C,mCAAmC,EAAE,UAAU;QAC/C,gCAAgC,EAAE,MAAM;QACxC,iDAAiD,EAAE,MAAM;QACzD,qDAAqD,EAAE,OAAO;QAC9D,qCAAqC,EAAE,MAAM;KAC9C,CAAC;IAEF,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;IAC7D,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAChC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IAEtB,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC;IACxC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;IAE5E,MAAM,IAAI,GAAG,uBAAuB,CAAC;QACnC,GAAG;QACH,OAAO,EAAE,WAAW;QACpB,cAAc,EAAE;YACd,GAAG;YACH,OAAO,EAAE,WAAW;YACpB,yBAAyB,EAAE;gBACzB,wBAAwB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;oBACrC,IAAI,EAAE,mBAAmB;oBACzB,cAAc,EAAE,cAAc;oBAC9B,wBAAwB,EAAE,SAAS;oBACnC,gCAAgC,EAAE,MAAM;oBACxC,UAAU,EAAE,MAAM;oBAClB,MAAM,EAAE,4FAA4F;oBACpG,eAAe,EAAE;wBACf,OAAO,EAAE,4FAA4F;wBACrG,KAAK,EAAE,CAAC,uDAAuD,CAAC;qBACjE;oBACD,qBAAqB,EAAE;wBACrB;4BACE,OAAO,EAAE,eAAe;4BACxB,gBAAgB,EAAE,QAAQ;4BAC1B,UAAU,EAAE,MAAM;4BAClB,uBAAuB,EAAE,MAAM;4BAC/B,MAAM,EAAE,uGAAuG;yBAChH;qBACF;iBACF,CAAC;aACH;SACF;KACF,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;QACpC,GAAG,EAAE,QAAQ;QACb,MAAM;QACN,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC7C,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,QAAQ;QAClB,YAAY,EAAE,sBAAsB;QACpC,aAAa,EAAE,kEAAkE;QACjF,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;QACxC,SAAS,EAAE,kBAAkB;QAC7B,GAAG,EAAE,QAAQ;QACb,MAAM;KACP,CAAC,CAAC;IACH,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;IAEvC,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,IAAI,mBAAmB,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,IAAI,qBAAqB,CAAC,EAAE,CAAC,CAAC;IACjD,MAAM,YAAY,GAAG,IAAI,8BAA8B,CAAC,EAAE,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,+BAA+B,CAAC,EAAE,CAAC,CAAC;IAE1D,MAAM,OAAO,GAAG,WAAW,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;IACrE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,YAAY,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IACtE,MAAM,MAAM,GAAmC;QAC7C,WAAW;QACX,QAAQ;QACR,MAAM;QACN,SAAS,EAAE,kBAAkB;QAC7B,SAAS;QACT,WAAW;QACX,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,YAAY,EAAE,eAAe;QAC7B,MAAM,EAAE;YACN,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB;QACD,UAAU,EAAE;YACV,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,QAAQ,EAAE,UAAU,CAAC,QAAQ;SAC9B;QACD,QAAQ,EAAE;YACR,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,aAAa,EAAE,SAAS,CAAC,aAAa;YACtC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;SACjD;QACD,WAAW,EAAE;YACX,YAAY,EAAE,WAAW,CAAC,KAAK,EAAE;YACjC,mBAAmB,EAAE,aAAa,CAAC,KAAK,EAAE;YAC1C,gBAAgB,EAAE,UAAU,CAAC,KAAK,EAAE;YACpC,mBAAmB,EAAE,YAAY,CAAC,KAAK,EAAE;YACzC,gBAAgB,EAAE,SAAS,CAAC,KAAK,EAAE;YACnC,gBAAgB,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC;SAChE;QACD,IAAI,EAAE;YACJ,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,aAAa,EAAE,IAAI,CAAC,cAAc,IAAI,UAAU;YAChD,UAAU,EAAE,IAAI,CAAC,WAAW;YAC5B,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,mBAAmB,EAAE,IAAI,CAAC,qBAAqB;SAChD;KACF,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;IACzD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;IAC3D,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzD,aAAa,CAAC,YAAY,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;IAE5E,OAAO;QACL,SAAS;QACT,QAAQ;QACR,YAAY;QACZ,MAAM;KACP,CAAC;AACJ,CAAC,CAAC"}

package/dist/evaluation/hybrid-phase1-rollout-summary.d.ts

@@ -0,0 +1,63 @@
+import type { HybridInvocationTrace, HybridReviewArtifact } from "../types/domain.js";
+export type HybridPhase1ReleaseRecommendation = "blocked" | "shadow_only" | "canary_ready" | "live_ready";
+export type HybridPhase1RolloutSummary = {
+    routeDistribution: Record<string, number>;
+    syncEscalationRate: number;
+    asyncReviewSchedulingRate: number;
+    workerOutputValidityRate: number;
+    fallbackRate: number;
+    explanationQualitySummary: {
+        surfaced: number;
+        shadowed: number;
+        fallbacks: number;
+    };
+    postmortemQualitySummary: {
+        storedArtifacts: number;
+        policyGatedArtifacts: number;
+        rejectedRuns: number;
+    };
+    phase2ExplainSummary?: {
+        llmBackedAttempts: number;
+        llmBackedFallbacks: number;
+        recommendation: "blocked" | "shadow_only" | "canary_ready" | "live_ready";
+    };
+    phase3PostmortemSummary?: {
+        llmBackedAttempts: number;
+        llmBackedFallbacks: number;
+        recommendation: "blocked" | "shadow_only" | "canary_ready" | "live_ready";
+    };
+    releaseGate?: {
+        stage: "offline" | "shadow" | "canary";
+        routeGatePassed: boolean;
+        explainGatePassed: boolean;
+        postmortemGatePassed: boolean;
+        runtimeGuardrailsPassed: boolean;
+    };
+    recommendation: HybridPhase1ReleaseRecommendation;
+};
+export declare const buildHybridPhase1RolloutSummary: (input: {
+    traces: HybridInvocationTrace[];
+    artifacts: HybridReviewArtifact[];
+    releaseGate?: {
+        stage: "offline" | "shadow" | "canary";
+        routeGatePassed: boolean;
+        explainGatePassed: boolean;
+        postmortemGatePassed: boolean;
+        runtimeGuardrailsPassed: boolean;
+    };
+    phase2ExplainGate?: {
+        stage: "offline" | "shadow" | "canary";
+        explainFaithfulnessPassed: boolean;
+        explainFallbackRatePassed: boolean;
+        explainTimeoutRatePassed: boolean;
+    };
+    phase3PostmortemGate?: {
+        stage: "offline" | "shadow" | "canary";
+        schemaValidOutputRatePassed: boolean;
+        timeoutFallbackRatePassed: boolean;
+        providerUnavailableFallbackRatePassed: boolean;
+        blockedClassificationStabilityPassed: boolean;
+        artifactSpamRatePassed: boolean;
+        backlogGrowthPassed: boolean;
+    };
+}) => HybridPhase1RolloutSummary;

package/dist/evaluation/hybrid-phase1-rollout-summary.js

@@ -0,0 +1,108 @@
+const ratio = (value, total) => total > 0 ? Number((value / total).toFixed(4)) : 0;
+export const buildHybridPhase1RolloutSummary = (input) => {
+    const routeDistribution = input.traces.reduce((acc, trace) => {
+        acc[trace.route] = (acc[trace.route] ?? 0) + 1;
+        return acc;
+    }, {});
+    const totalTraces = input.traces.length;
+    const syncEscalations = input.traces.filter((trace) => trace.route === "ESCALATE_SYNC_EXPLAIN").length;
+    const asyncReviews = input.traces.filter((trace) => trace.route === "ESCALATE_ASYNC_POSTMORTEM").length;
+    const accepted = input.traces.filter((trace) => trace.validation_status === "accepted").length;
+    const fallbacks = input.traces.filter((trace) => trace.validation_status === "fallback").length;
+    const explainTraces = input.traces.filter((trace) => trace.worker_task === "explain_decision");
+    const postmortemTraces = input.traces.filter((trace) => trace.worker_task === "postmortem_review");
+    const explanationQualitySummary = {
+        surfaced: explainTraces.filter((trace) => trace.output_action === "surfaced").length,
+        shadowed: explainTraces.filter((trace) => trace.output_action === "none").length,
+        fallbacks: explainTraces.filter((trace) => trace.validation_status === "fallback").length
+    };
+    const postmortemQualitySummary = {
+        storedArtifacts: postmortemTraces.filter((trace) => trace.output_action === "stored").length,
+        policyGatedArtifacts: input.artifacts.filter((artifact) => artifact.approval_class === "policy_gated").length,
+        rejectedRuns: postmortemTraces.filter((trace) => trace.output_action === "rejected").length
+    };
+    const llmExplainTraces = explainTraces.filter((trace) => trace.worker_profile_version?.startsWith("hybrid-explain-llm"));
+    const phase2ExplainGate = input.phase2ExplainGate;
+    let phase2ExplainSummary;
+    if (phase2ExplainGate) {
+        const explainGatesPassed = phase2ExplainGate.explainFaithfulnessPassed
+            && phase2ExplainGate.explainFallbackRatePassed
+            && phase2ExplainGate.explainTimeoutRatePassed;
+        let recommendation = "blocked";
+        if (explainGatesPassed) {
+            if (phase2ExplainGate.stage === "offline") {
+                recommendation = "shadow_only";
+            }
+            else if (phase2ExplainGate.stage === "shadow") {
+                recommendation = "canary_ready";
+            }
+            else if (phase2ExplainGate.stage === "canary") {
+                recommendation = "live_ready";
+            }
+        }
+        phase2ExplainSummary = {
+            llmBackedAttempts: llmExplainTraces.length,
+            llmBackedFallbacks: llmExplainTraces.filter((trace) => trace.validation_status === "fallback").length,
+            recommendation
+        };
+    }
+    const llmPostmortemTraces = postmortemTraces.filter((trace) => trace.worker_profile_version?.startsWith("hybrid-postmortem-llm"));
+    const phase3PostmortemGate = input.phase3PostmortemGate;
+    let phase3PostmortemSummary;
+    if (phase3PostmortemGate) {
+        const postmortemGatesPassed = phase3PostmortemGate.schemaValidOutputRatePassed
+            && phase3PostmortemGate.timeoutFallbackRatePassed
+            && phase3PostmortemGate.providerUnavailableFallbackRatePassed
+            && phase3PostmortemGate.blockedClassificationStabilityPassed
+            && phase3PostmortemGate.artifactSpamRatePassed
+            && phase3PostmortemGate.backlogGrowthPassed;
+        let recommendation = "blocked";
+        if (postmortemGatesPassed) {
+            if (phase3PostmortemGate.stage === "offline") {
+                recommendation = "shadow_only";
+            }
+            else if (phase3PostmortemGate.stage === "shadow") {
+                recommendation = "canary_ready";
+            }
+            else if (phase3PostmortemGate.stage === "canary") {
+                recommendation = "live_ready";
+            }
+        }
+        phase3PostmortemSummary = {
+            llmBackedAttempts: llmPostmortemTraces.length,
+            llmBackedFallbacks: llmPostmortemTraces.filter((trace) => trace.validation_status === "fallback").length,
+            recommendation
+        };
+    }
+    const gate = input.releaseGate;
+    let recommendation = "blocked";
+    const gatesPassed = gate?.routeGatePassed === true
+        && gate.explainGatePassed === true
+        && gate.postmortemGatePassed === true
+        && gate.runtimeGuardrailsPassed === true;
+    if (gatesPassed && totalTraces > 0) {
+        if (gate?.stage === "offline") {
+            recommendation = "shadow_only";
+        }
+        else if (gate?.stage === "shadow") {
+            recommendation = "canary_ready";
+        }
+        else if (gate?.stage === "canary") {
+            recommendation = "live_ready";
+        }
+    }
+    return {
+        routeDistribution,
+        syncEscalationRate: ratio(syncEscalations, totalTraces),
+        asyncReviewSchedulingRate: ratio(asyncReviews, totalTraces),
+        workerOutputValidityRate: ratio(accepted, totalTraces),
+        fallbackRate: ratio(fallbacks, totalTraces),
+        explanationQualitySummary,
+        postmortemQualitySummary,
+        phase2ExplainSummary,
+        phase3PostmortemSummary,
+        releaseGate: gate,
+        recommendation
+    };
+};
+//# sourceMappingURL=hybrid-phase1-rollout-summary.js.map

package/dist/evaluation/hybrid-phase1-rollout-summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-phase1-rollout-summary.js","sourceRoot":"","sources":["../../src/evaluation/hybrid-phase1-rollout-summary.ts"],"names":[],"mappings":"AAwCA,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE,CACrD,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAErD,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,KAyB/C,EAA8B,EAAE;IAC/B,MAAM,iBAAiB,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAyB,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QACnF,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACxC,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,uBAAuB,CAAC,CAAC,MAAM,CAAC;IACvG,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,2BAA2B,CAAC,CAAC,MAAM,CAAC;IACxG,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAC/F,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAEhG,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,kBAAkB,CAAC,CAAC;IAC/F,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,mBAAmB,CAAC,CAAC;IAEnG,MAAM,yBAAyB,GAAG;QAChC,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,UAAU,CAAC,CAAC,MAAM;QACpF,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC,MAAM;QAChF,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,MAAM;KAC1F,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,eAAe,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC5F,oBAAoB,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,cAAc,KAAK,cAAc,CAAC,CAAC,MAAM;QAC7G,YAAY,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,UAAU,CAAC,CAAC,MAAM;KAC5F,CAAC;IAEF,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,sBAAsB,EAAE,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACzH,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;IAClD,IAAI,oBAAwE,CAAC;IAC7E,IAAI,iBAAiB,EAAE,CAAC;QACtB,MAAM,kBAAkB,GACtB,iBAAiB,CAAC,yBAAyB;eACxC,iBAAiB,CAAC,yBAAyB;eAC3C,iBAAiB,CAAC,wBAAwB,CAAC;QAChD,IAAI,cAAc,GAAsC,SAAS,CAAC;QAClE,IAAI,kBAAkB,EAAE,CAAC;YACvB,IAAI,iBAAiB,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC1C,cAAc,GAAG,aAAa,CAAC;YACjC,CAAC;iBAAM,IAAI,iBAAiB,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAChD,cAAc,GAAG,cAAc,CAAC;YAClC,CAAC;iBAAM,IAAI,iBAAiB,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAChD,cAAc,GAAG,YAAY,CAAC;YAChC,CAAC;QACH,CAAC;QACD,oBAAoB,GAAG;YACrB,iBAAiB,EAAE,gBAAgB,CAAC,MAAM;YAC1C,kBAAkB,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,MAAM;YACrG,cAAc;SACf,CAAC;IACJ,CAAC;IAED,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAC5D,KAAK,CAAC,sBAAsB,EAAE,UAAU,CAAC,uBAAuB,CAAC,CAClE,CAAC;IACF,MAAM,oBAAoB,GAAG,KAAK,CAAC,oBAAoB,CAAC;IACxD,IAAI,uBAA8E,CAAC;IACnF,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,qBAAqB,GACzB,oBAAoB,CAAC,2BAA2B;eAC7C,oBAAoB,CAAC,yBAAyB;eAC9C,oBAAoB,CAAC,qCAAqC;eAC1D,oBAAoB,CAAC,oCAAoC;eACzD,oBAAoB,CAAC,sBAAsB;eAC3C,oBAAoB,CAAC,mBAAmB,CAAC;QAC9C,IAAI,cAAc,GAAsC,SAAS,CAAC;QAClE,IAAI,qBAAqB,EAAE,CAAC;YAC1B,IAAI,oBAAoB,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7C,cAAc,GAAG,aAAa,CAAC;YACjC,CAAC;iBAAM,IAAI,oBAAoB,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACnD,cAAc,GAAG,cAAc,CAAC;YAClC,CAAC;iBAAM,IAAI,oBAAoB,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACnD,cAAc,GAAG,YAAY,CAAC;YAChC,CAAC;QACH,CAAC;QACD,uBAAuB,GAAG;YACxB,iBAAiB,EAAE,mBAAmB,CAAC,MAAM;YAC7C,kBAAkB,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,MAAM;YACxG,cAAc;SACf,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC;IAC/B,IAAI,cAAc,GAAsC,SAAS,CAAC;IAClE,MAAM,WAAW,GACf,IAAI,EAAE,eAAe,KAAK,IAAI;WAC3B,IAAI,CAAC,iBAAiB,KAAK,IAAI;WAC/B,IAAI,CAAC,oBAAoB,KAAK,IAAI;WAClC,IAAI,CAAC,uBAAuB,KAAK,IAAI,CAAC;IAC3C,IAAI,WAAW,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,IAAI,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,cAAc,GAAG,aAAa,CAAC;QACjC,CAAC;aAAM,IAAI,IAAI,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,GAAG,cAAc,CAAC;QAClC,CAAC;aAAM,IAAI,IAAI,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,GAAG,YAAY,CAAC;QAChC,CAAC;IACH,CAAC;IAED,OAAO;QACL,iBAAiB;QACjB,kBAAkB,EAAE,KAAK,CAAC,eAAe,EAAE,WAAW,CAAC;QACvD,yBAAyB,EAAE,KAAK,CAAC,YAAY,EAAE,WAAW,CAAC;QAC3D,wBAAwB,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,CAAC;QACtD,YAAY,EAAE,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC;QAC3C,yBAAyB;QACzB,wBAAwB;QACxB,oBAAoB;QACpB,uBAAuB;QACvB,WAAW,EAAE,IAAI;QACjB,cAAc;KACf,CAAC;AACJ,CAAC,CAAC"}

package/dist/evaluation/hybrid-phase3-gate-metrics.d.ts

@@ -0,0 +1,26 @@
+export type HybridPhase3GateMetrics = {
+    schemaValidOutputRate: number;
+    timeoutFallbackRate: number;
+    providerUnavailableFallbackRate: number;
+    blockedPolicyGatedStability: number;
+    falsePositiveRecommendationRate: number;
+    artifactSpamRate: number;
+    backlogGrowthVsBaseline: number;
+};
+export declare const buildHybridPhase3GateMetrics: (input: {
+    scheduledEligibleRuns: number;
+    acceptedArtifacts: number;
+    policyGatedArtifacts: number;
+    blockedOutputs: number;
+    timeoutFallbacks: number;
+    providerUnavailableFallbacks: number;
+    validationFailedFallbacks: number;
+    falsePositiveRecommendations: number;
+    deterministicBaseline: {
+        eligibleRuns: number;
+        backlogSize: number;
+    };
+    currentWindow: {
+        backlogSize: number;
+    };
+}) => HybridPhase3GateMetrics;

package/dist/evaluation/hybrid-phase3-gate-metrics.js

@@ -0,0 +1,23 @@
+const ratio = (value, total) => total > 0 ? Number((value / total).toFixed(4)) : 0;
+export const buildHybridPhase3GateMetrics = (input) => {
+    const totalAttempts = input.acceptedArtifacts
+        + input.policyGatedArtifacts
+        + input.blockedOutputs
+        + input.timeoutFallbacks
+        + input.providerUnavailableFallbacks
+        + input.validationFailedFallbacks;
+    const acceptedLike = input.acceptedArtifacts + input.policyGatedArtifacts;
+    const baselineBacklog = input.deterministicBaseline.backlogSize;
+    const currentBacklog = input.currentWindow.backlogSize;
+    const backlogGrowthVsBaseline = baselineBacklog > 0 ? Number(((currentBacklog - baselineBacklog) / baselineBacklog).toFixed(4)) : 0;
+    return {
+        schemaValidOutputRate: ratio(acceptedLike + input.blockedOutputs, totalAttempts),
+        timeoutFallbackRate: ratio(input.timeoutFallbacks, totalAttempts),
+        providerUnavailableFallbackRate: ratio(input.providerUnavailableFallbacks, totalAttempts),
+        blockedPolicyGatedStability: ratio(acceptedLike + input.blockedOutputs, totalAttempts),
+        falsePositiveRecommendationRate: ratio(input.falsePositiveRecommendations, totalAttempts),
+        artifactSpamRate: ratio(acceptedLike, input.scheduledEligibleRuns),
+        backlogGrowthVsBaseline
+    };
+};
+//# sourceMappingURL=hybrid-phase3-gate-metrics.js.map

package/dist/evaluation/hybrid-phase3-gate-metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-phase3-gate-metrics.js","sourceRoot":"","sources":["../../src/evaluation/hybrid-phase3-gate-metrics.ts"],"names":[],"mappings":"AAUA,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE,CACrD,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAErD,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,KAgB5C,EAA2B,EAAE;IAC5B,MAAM,aAAa,GACjB,KAAK,CAAC,iBAAiB;UACrB,KAAK,CAAC,oBAAoB;UAC1B,KAAK,CAAC,cAAc;UACpB,KAAK,CAAC,gBAAgB;UACtB,KAAK,CAAC,4BAA4B;UAClC,KAAK,CAAC,yBAAyB,CAAC;IACpC,MAAM,YAAY,GAAG,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,oBAAoB,CAAC;IAC1E,MAAM,eAAe,GAAG,KAAK,CAAC,qBAAqB,CAAC,WAAW,CAAC;IAChE,MAAM,cAAc,GAAG,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC;IACvD,MAAM,uBAAuB,GAC3B,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,GAAG,eAAe,CAAC,GAAG,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtG,OAAO;QACL,qBAAqB,EAAE,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC;QAChF,mBAAmB,EAAE,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,aAAa,CAAC;QACjE,+BAA+B,EAAE,KAAK,CAAC,KAAK,CAAC,4BAA4B,EAAE,aAAa,CAAC;QACzF,2BAA2B,EAAE,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC;QACtF,+BAA+B,EAAE,KAAK,CAAC,KAAK,CAAC,4BAA4B,EAAE,aAAa,CAAC;QACzF,gBAAgB,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,qBAAqB,CAAC;QAClE,uBAAuB;KACxB,CAAC;AACJ,CAAC,CAAC"}

package/dist/evaluation/openclaw-baseline.d.ts

@@ -63,6 +63,14 @@ export type OpenClawBaselineSummary = {
         automaticHelped: number;
         automaticHarmed: number;
     };
+    governance: {
+        harmfulOrMisfiredHints: number;
+        harmfulOrMisfiredRate: number;
+        metaDominantSelections: number;
+        metaDominantRate: number;
+        realDevAlignedSelections: number;
+        realDevAlignedRate: number;
+    };
     benchmark: BenchmarkSummary;
     trend?: OpenClawBaselineTrend;
     modeComparison: {

package/dist/evaluation/openclaw-baseline.js

@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { loadConfig } from "../config/load-config.js";
 import { resolveExperienceEnginePaths } from "../config/path-resolver.js";
+import { deriveGovernanceSignals, isPotentialMisfire, parseInjectionScorecard } from "../experience-management/governance-observability.js";
 import { openDatabase } from "../store/sqlite/db.js";
 import { InjectionRepository } from "../store/sqlite/repositories/injection-repo.js";
 import { buildBenchmarkSummary, buildModeBenchmarkSummary } from "./benchmark-summary.js";
@@ -145,6 +146,15 @@ export const collectOpenClawBaselineSummary = (db, config, options = {}) => {
             automaticHarmed: injectionRepo.countAutomaticFeedbackByDeliveryMode("holdout", "mark_harmed")
         })
     };
+    const governanceRows = db.prepare(`SELECT scorecard_json, harm_observed, attribution_reason
+     FROM injection_events ${recordFilter}`).all();
+    const harmfulOrMisfiredHints = governanceRows.filter((row) => isPotentialMisfire({
+        harm_observed: row.harm_observed == null ? null : Boolean(row.harm_observed),
+        attribution_reason: row.attribution_reason ?? undefined
+    })).length;
+    const governanceSignals = governanceRows.map((row) => deriveGovernanceSignals(parseInjectionScorecard(row.scorecard_json)));
+    const metaDominantSelections = governanceSignals.filter((signal) => signal.metaDominant).length;
+    const realDevAlignedSelections = governanceSignals.filter((signal) => signal.realDevAligned).length;
     const attributionReasons = [
         "success_outcome",
         "relevant_failure",
@@ -219,6 +229,14 @@ export const collectOpenClawBaselineSummary = (db, config, options = {}) => {
             automaticHelped,
             automaticHarmed
         },
+        governance: {
+            harmfulOrMisfiredHints,
+            harmfulOrMisfiredRate: ratio(harmfulOrMisfiredHints, injectionTotal),
+            metaDominantSelections,
+            metaDominantRate: ratio(metaDominantSelections, injectionTotal),
+            realDevAlignedSelections,
+            realDevAlignedRate: ratio(realDevAlignedSelections, injectionTotal)
+        },
         benchmark: buildBenchmarkSummary({
             decisions: injectionTotal,
             live: liveDecisions,
@@ -316,6 +334,15 @@ export const renderOpenClawBaselineMarkdown = (summary) => `# OpenClaw Baseline
 - Automatic helped: ${summary.effectiveness.automaticHelped}
 - Automatic harmed: ${summary.effectiveness.automaticHarmed}
 
+## Governance
+
+- Harmful or misfired hints: ${summary.governance.harmfulOrMisfiredHints}
+- Harmful or misfired rate: ${summary.governance.harmfulOrMisfiredRate}
+- Meta-dominant selections: ${summary.governance.metaDominantSelections}
+- Meta-dominant rate: ${summary.governance.metaDominantRate}
+- Real-dev-aligned selections: ${summary.governance.realDevAlignedSelections}
+- Real-dev-aligned rate: ${summary.governance.realDevAlignedRate}
+
 ## Benchmark Summary
 
 - Delivery rate: ${summary.benchmark.deliveryRate}