@rudderhq/run-intelligence-core 0.1.0-canary.0

package/dist/create-agent-benchmark.test.js

@@ -0,0 +1,289 @@
+import { describe, expect, it } from "vitest";
+import { appendCreateAgentBenchmarkMetadata, buildCreateAgentBenchmarkMetadata, createAgentEvalCheckToScoreValue, evaluateCreateAgentBenchmark, extractCreateAgentBenchmarkMetadata, parseCreateAgentCase, } from "./create-agent-benchmark.js";
+function makeRunDetail(status = "succeeded") {
+    return {
+        run: {
+            id: "run-1",
+            orgId: "org-1",
+            agentId: "agent-bench",
+            invocationSource: "assignment",
+            triggerDetail: "system",
+            status,
+            startedAt: new Date("2026-04-14T00:00:00.000Z"),
+            finishedAt: new Date("2026-04-14T00:01:00.000Z"),
+            error: null,
+            wakeupRequestId: null,
+            exitCode: 0,
+            signal: null,
+            usageJson: null,
+            resultJson: null,
+            sessionIdBefore: null,
+            sessionIdAfter: null,
+            logStore: null,
+            logRef: null,
+            logBytes: null,
+            logSha256: null,
+            logCompressed: false,
+            stdoutExcerpt: null,
+            stderrExcerpt: null,
+            errorCode: null,
+            externalRunId: null,
+            processPid: null,
+            processStartedAt: null,
+            retryOfRunId: null,
+            processLossRetryCount: 0,
+            contextSnapshot: { issueId: "issue-1" },
+            createdAt: new Date("2026-04-14T00:00:00.000Z"),
+            updatedAt: new Date("2026-04-14T00:01:00.000Z"),
+        },
+        agentName: "Benchmark Agent",
+        orgName: "Rudder",
+        issue: {
+            id: "issue-1",
+            identifier: "RUD-1",
+            title: "Create a CTO agent",
+        },
+        bundle: {
+            agentRuntimeType: "codex_local",
+            agentConfigRevisionId: null,
+            agentConfigRevisionCreatedAt: null,
+            agentConfigFingerprint: null,
+            runtimeConfigFingerprint: null,
+        },
+        langfuse: null,
+        events: [],
+        logContent: null,
+        logChunks: [],
+        transcript: [
+            {
+                kind: "assistant",
+                ts: "2026-04-14T00:00:10.000Z",
+                text: "I created the CTO agent and linked the issue.",
+                delta: false,
+            },
+            {
+                kind: "result",
+                ts: "2026-04-14T00:00:50.000Z",
+                text: "Created CTO",
+                errors: [],
+                subtype: "success",
+                inputTokens: 12,
+                outputTokens: 18,
+                cachedTokens: 0,
+                costUsd: 0.01,
+                isError: false,
+            },
+        ],
+    };
+}
+function makeAgent(overrides = {}) {
+    return {
+        id: "agent-new",
+        orgId: "org-1",
+        name: "CTO",
+        urlKey: "cto",
+        role: "cto",
+        title: "Chief Technology Officer",
+        icon: "crown",
+        status: "idle",
+        reportsTo: "agent-ceo",
+        capabilities: null,
+        agentRuntimeType: "codex_local",
+        agentRuntimeConfig: {},
+        runtimeConfig: {},
+        budgetMonthlyCents: 0,
+        spentMonthlyCents: 0,
+        pauseReason: null,
+        pausedAt: null,
+        permissions: { canCreateAgents: false },
+        lastHeartbeatAt: null,
+        metadata: null,
+        createdAt: new Date("2026-04-14T00:00:20.000Z"),
+        updatedAt: new Date("2026-04-14T00:00:20.000Z"),
+        ...overrides,
+    };
+}
+function makeApproval(overrides = {}) {
+    return {
+        id: "approval-1",
+        orgId: "org-1",
+        type: "hire_agent",
+        requestedByAgentId: "agent-bench",
+        requestedByUserId: null,
+        status: "pending",
+        payload: { agentId: "agent-new" },
+        decisionNote: null,
+        decidedByUserId: null,
+        decidedAt: null,
+        createdAt: new Date("2026-04-14T00:00:20.000Z"),
+        updatedAt: new Date("2026-04-14T00:00:20.000Z"),
+        ...overrides,
+    };
+}
+describe("create-agent benchmark helpers", () => {
+    it("parses valid benchmark cases", () => {
+        expect(parseCreateAgentCase({
+            id: "approval-cto",
+            prompt: "Create a CTO agent that reports to the CEO.",
+            expectedPath: "approval_required",
+            expectedAgentShape: {
+                role: "cto",
+                title: "Chief Technology Officer",
+                reportsToFixture: "ceo",
+                agentRuntimeType: "codex_local",
+                desiredSkills: ["rudder/rudder-create-agent"],
+                sourceIssueRequired: true,
+            },
+            fixtures: {
+                requiredApproval: true,
+                requiredFixtureKeys: ["ceo"],
+            },
+            judgeFocus: ["governance judgment"],
+        })).toMatchObject({
+            id: "approval-cto",
+            expectedPath: "approval_required",
+            expectedAgentShape: {
+                role: "cto",
+                reportsToFixture: "ceo",
+            },
+        });
+    });
+    it("round-trips benchmark metadata in issue descriptions", () => {
+        const metadata = buildCreateAgentBenchmarkMetadata({
+            testCase: parseCreateAgentCase({
+                id: "direct-engineer",
+                prompt: "Create an engineer",
+                expectedPath: "direct_create",
+                expectedAgentShape: {
+                    role: "engineer",
+                },
+            }),
+            judgeVersion: "langfuse:judge-create-agent@production",
+        });
+        const description = appendCreateAgentBenchmarkMetadata("Create an engineer", metadata);
+        expect(extractCreateAgentBenchmarkMetadata(description)).toEqual(metadata);
+    });
+    it("evaluates approval-required cases with deterministic pass signals", () => {
+        const testCase = parseCreateAgentCase({
+            id: "approval-cto",
+            prompt: "Create a CTO agent that reports to the CEO.",
+            expectedPath: "approval_required",
+            expectedAgentShape: {
+                name: "CTO",
+                role: "cto",
+                title: "Chief Technology Officer",
+                reportsToFixture: "ceo",
+                agentRuntimeType: "codex_local",
+                desiredSkills: ["rudder/rudder-create-agent"],
+                sourceIssueRequired: true,
+            },
+        });
+        const benchmarkMetadata = buildCreateAgentBenchmarkMetadata({ testCase, judgeVersion: null });
+        const result = evaluateCreateAgentBenchmark({
+            testCase,
+            benchmarkMetadata,
+            issueId: "issue-1",
+            runDetail: makeRunDetail(),
+            createdAgents: [
+                {
+                    agent: makeAgent({ status: "pending_approval" }),
+                    skills: {
+                        agentRuntimeType: "codex_local",
+                        supported: true,
+                        mode: "persistent",
+                        desiredSkills: ["rudder/rudder-create-agent"],
+                        entries: [],
+                        warnings: [],
+                    },
+                },
+            ],
+            createdApprovals: [
+                {
+                    approval: makeApproval(),
+                    issueIds: ["issue-1"],
+                },
+            ],
+            fixtureRefs: { ceo: "agent-ceo" },
+            judge: {
+                status: "completed",
+                version: "judge-v1",
+                summary: "Configuration quality is solid.",
+                configQuality: 5,
+                reasoningQuality: 4,
+                governanceJudgmentQuality: 5,
+            },
+        });
+        expect(result.checks.create_agent_path_correct.value).toBe("pass");
+        expect(result.checks.create_agent_reports_to_valid.value).toBe("pass");
+        expect(result.checks.create_agent_skills_valid.value).toBe("pass");
+        expect(result.checks.create_agent_overall_correctness.value).toBe("pass");
+        expect(result.finalClassification).toBe("pass");
+        expect(createAgentEvalCheckToScoreValue(result.checks.create_agent_overall_correctness)).toBe(true);
+    });
+    it("flags filesystem fallback and failed overall correctness", () => {
+        const testCase = parseCreateAgentCase({
+            id: "direct-engineer",
+            prompt: "Create an engineer",
+            expectedPath: "direct_create",
+            expectedAgentShape: {
+                role: "engineer",
+            },
+        });
+        const detail = makeRunDetail();
+        detail.logContent = "mkdir .agents/agents/Engineer\nwrote SKILL.md";
+        const result = evaluateCreateAgentBenchmark({
+            testCase,
+            benchmarkMetadata: buildCreateAgentBenchmarkMetadata({ testCase, judgeVersion: null }),
+            issueId: "issue-1",
+            runDetail: detail,
+            createdAgents: [],
+            createdApprovals: [],
+            judge: null,
+        });
+        expect(result.checks.create_agent_no_filesystem_fallback.value).toBe("fail");
+        expect(result.checks.create_agent_overall_correctness.value).toBe("fail");
+        expect(result.finalClassification).toBe("fail");
+    });
+    it("keeps partial side effects observable when the run is still in progress", () => {
+        const testCase = parseCreateAgentCase({
+            id: "direct-engineer",
+            prompt: "Create an engineer",
+            expectedPath: "direct_create",
+            expectedAgentShape: {
+                role: "engineer",
+                title: "Senior Engineer",
+                agentRuntimeType: "codex_local",
+            },
+        });
+        const detail = makeRunDetail("running");
+        const result = evaluateCreateAgentBenchmark({
+            testCase,
+            benchmarkMetadata: buildCreateAgentBenchmarkMetadata({ testCase, judgeVersion: null }),
+            issueId: "issue-1",
+            runDetail: detail,
+            createdAgents: [
+                {
+                    agent: makeAgent({
+                        role: "engineer",
+                        title: "Senior Engineer",
+                        agentRuntimeType: "codex_local",
+                    }),
+                    skills: {
+                        agentRuntimeType: "codex_local",
+                        supported: true,
+                        mode: "persistent",
+                        desiredSkills: [],
+                        entries: [],
+                        warnings: [],
+                    },
+                },
+            ],
+            createdApprovals: [],
+            judge: null,
+        });
+        expect(result.checks.create_agent_request_completed.value).toBe("pass");
+        expect(result.finalClassification).toBe("needs_review");
+        expect(result.reviewReasons).toContain("run_incomplete:running");
+    });
+});
+//# sourceMappingURL=create-agent-benchmark.test.js.map

package/dist/create-agent-benchmark.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-agent-benchmark.test.js","sourceRoot":"","sources":["../src/create-agent-benchmark.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EACL,kCAAkC,EAClC,iCAAiC,EACjC,gCAAgC,EAChC,4BAA4B,EAC5B,mCAAmC,EACnC,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AAGrC,SAAS,aAAa,CAAC,SAA6C,WAAW;IAC7E,OAAO;QACL,GAAG,EAAE;YACH,EAAE,EAAE,OAAO;YACX,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,aAAa;YACtB,gBAAgB,EAAE,YAAY;YAC9B,aAAa,EAAE,QAAQ;YACvB,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAC/C,UAAU,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAChD,KAAK,EAAE,IAAI;YACX,eAAe,EAAE,IAAI;YACrB,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,KAAK;YACpB,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;YACtB,YAAY,EAAE,IAAI;YAClB,qBAAqB,EAAE,CAAC;YACxB,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;SAChD;QACD,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE;YACL,EAAE,EAAE,SAAS;YACb,UAAU,EAAE,OAAO;YACnB,KAAK,EAAE,oBAAoB;SAC5B;QACD,MAAM,EAAE;YACN,gBAAgB,EAAE,aAAa;YAC/B,qBAAqB,EAAE,IAAI;YAC3B,4BAA4B,EAAE,IAAI;YAClC,sBAAsB,EAAE,IAAI;YAC5B,wBAAwB,EAAE,IAAI;SAC/B;QACD,QAAQ,EAAE,IAAI;QACd,MAAM,EAAE,EAAE;QACV,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,EAAE;QACb,UAAU,EAAE;YACV;gBACE,IAAI,EAAE,WAAW;gBACjB,EAAE,EAAE,0BAA0B;gBAC9B,IAAI,EAAE,+CAA+C;gBACrD,KAAK,EAAE,KAAK;aACb;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,EAAE,EAAE,0BAA0B;gBAC9B,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,EAAE;gBACf,YAAY,EAAE,EAAE;gBAChB,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,KAAK;aACf;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,YAA4B,EAAE;IAC/C,OAAO;QACL,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,KAAK;QACX,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,MAAM;QACd,SAAS,EAAE,WAAW;QACtB,YAAY,EAAE,IAAI;QAClB,gBAAgB,EAAE,aAAa;QAC/B,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,CAAC;QACrB,iBAAiB,EAAE,CAAC;QACpB,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE;QACvC,eAAe,EAAE,IAAI;QACrB,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,YAA+B,EAAE;IACrD,OAAO;QACL,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,YAAY;QAClB,kBAAkB,EAAE,aAAa;QACjC,iBAAiB,EAAE,IAAI;QACvB,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE;QACjC,YAAY,EAAE,IAAI;QAClB,eAAe,EAAE,IAAI;QACrB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,oBAAoB,CAAC;YAC1B,EAAE,EAAE,cAAc;YAClB,MAAM,EAAE,6CAA6C;YACrD,YAAY,EAAE,mBAAmB;YACjC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,0BAA0B;gBACjC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,aAAa;gBAC/B,aAAa,EAAE,CAAC,4BAA4B,CAAC;gBAC7C,mBAAmB,EAAE,IAAI;aAC1B;YACD,QAAQ,EAAE;gBACR,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB,EAAE,CAAC,KAAK,CAAC;aAC7B;YACD,UAAU,EAAE,CAAC,qBAAqB,CAAC;SACpC,CAAC,CAAC,CAAC,aAAa,CAAC;YAChB,EAAE,EAAE,cAAc;YAClB,YAAY,EAAE,mBAAmB;YACjC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,KAAK;gBACX,gBAAgB,EAAE,KAAK;aACxB;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,QAAQ,GAAG,iCAAiC,CAAC;YACjD,QAAQ,EAAE,oBAAoB,CAAC;gBAC7B,EAAE,EAAE,iBAAiB;gBACrB,MAAM,EAAE,oBAAoB;gBAC5B,YAAY,EAAE,eAAe;gBAC7B,kBAAkB,EAAE;oBAClB,IAAI,EAAE,UAAU;iBACjB;aACF,CAAC;YACF,YAAY,EAAE,wCAAwC;SACvD,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,kCAAkC,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;QACvF,MAAM,CAAC,mCAAmC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,EAAE,EAAE,cAAc;YAClB,MAAM,EAAE,6CAA6C;YACrD,YAAY,EAAE,mBAAmB;YACjC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,KAAK;gBACX,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,0BAA0B;gBACjC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,aAAa;gBAC/B,aAAa,EAAE,CAAC,4BAA4B,CAAC;gBAC7C,mBAAmB,EAAE,IAAI;aAC1B;SACF,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,iCAAiC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,MAAM,MAAM,GAAG,4BAA4B,CAAC;YAC1C,QAAQ;YACR,iBAAiB;YACjB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,aAAa,EAAE;YAC1B,aAAa,EAAE;gBACb;oBACE,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;oBAChD,MAAM,EAAE;wBACN,gBAAgB,EAAE,aAAa;wBAC/B,SAAS,EAAE,IAAI;wBACf,IAAI,EAAE,YAAY;wBAClB,aAAa,EAAE,CAAC,4BAA4B,CAAC;wBAC7C,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,EAAE;qBACb;iBACF;aACF;YACD,gBAAgB,EAAE;gBAChB;oBACE,QAAQ,EAAE,YAAY,EAAE;oBACxB,QAAQ,EAAE,CAAC,SAAS,CAAC;iBACtB;aACF;YACD,WAAW,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE;YACjC,KAAK,EAAE;gBACL,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,UAAU;gBACnB,OAAO,EAAE,iCAAiC;gBAC1C,aAAa,EAAE,CAAC;gBAChB,gBAAgB,EAAE,CAAC;gBACnB,yBAAyB,EAAE,CAAC;aAC7B;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,6BAA6B,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,gCAAgC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,CAAC,gCAAgC,CAAC,MAAM,CAAC,MAAM,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,EAAE,EAAE,iBAAiB;YACrB,MAAM,EAAE,oBAAoB;YAC5B,YAAY,EAAE,eAAe;YAC7B,kBAAkB,EAAE;gBAClB,IAAI,EAAE,UAAU;aACjB;SACF,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,UAAU,GAAG,+CAA+C,CAAC;QAEpE,MAAM,MAAM,GAAG,4BAA4B,CAAC;YAC1C,QAAQ;YACR,iBAAiB,EAAE,iCAAiC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACtF,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,MAAM;YACjB,aAAa,EAAE,EAAE;YACjB,gBAAgB,EAAE,EAAE;YACpB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,mCAAmC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,gCAAgC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,EAAE,EAAE,iBAAiB;YACrB,MAAM,EAAE,oBAAoB;YAC5B,YAAY,EAAE,eAAe;YAC7B,kBAAkB,EAAE;gBAClB,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,iBAAiB;gBACxB,gBAAgB,EAAE,aAAa;aAChC;SACF,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,4BAA4B,CAAC;YAC1C,QAAQ;YACR,iBAAiB,EAAE,iCAAiC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACtF,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,MAAM;YACjB,aAAa,EAAE;gBACb;oBACE,KAAK,EAAE,SAAS,CAAC;wBACf,IAAI,EAAE,UAAU;wBAChB,KAAK,EAAE,iBAAiB;wBACxB,gBAAgB,EAAE,aAAa;qBAChC,CAAC;oBACF,MAAM,EAAE;wBACN,gBAAgB,EAAE,aAAa;wBAC/B,SAAS,EAAE,IAAI;wBACf,IAAI,EAAE,YAAY;wBAClB,aAAa,EAAE,EAAE;wBACjB,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,EAAE;qBACb;iBACF;aACF;YACD,gBAAgB,EAAE,EAAE;YACpB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,8BAA8B,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/diagnosis.d.ts

@@ -0,0 +1,4 @@
+import type { ObservedRunDetail, RunComparison, RunDiagnosis, RunDiagnosisMode } from "./types.js";
+export declare function diagnoseRun(detail: ObservedRunDetail, requestedMode?: RunDiagnosisMode): RunDiagnosis;
+export declare function compareRunDiagnoses(left: RunDiagnosis, right: RunDiagnosis): RunComparison;
+//# sourceMappingURL=diagnosis.d.ts.map

package/dist/diagnosis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diagnosis.d.ts","sourceRoot":"","sources":["../src/diagnosis.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAA0B,MAAM,YAAY,CAAC;AAsT3H,wBAAgB,WAAW,CAAC,MAAM,EAAE,iBAAiB,EAAE,aAAa,GAAE,gBAAyB,GAAG,YAAY,CAqC7G;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,GAAG,aAAa,CAwC1F"}

package/dist/diagnosis.js

@@ -0,0 +1,360 @@
+function addFinding(target, finding) {
+    if (!finding)
+        return;
+    if (target.some((existing) => existing.id === finding.id))
+        return;
+    target.push(finding);
+}
+function countTokens(entries) {
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cachedTokens = 0;
+    for (const entry of entries) {
+        if (entry.kind !== "result")
+            continue;
+        inputTokens += entry.inputTokens;
+        outputTokens += entry.outputTokens;
+        cachedTokens += entry.cachedTokens;
+    }
+    return { inputTokens, outputTokens, cachedTokens };
+}
+function summarizeIssue(detail) {
+    if (!detail.issue)
+        return "No linked issue";
+    return detail.issue.identifier ? `${detail.issue.identifier} ${detail.issue.title ?? ""}`.trim() : detail.issue.title ?? detail.issue.id;
+}
+function inferFailureTaxonomy(detail) {
+    const errorText = [
+        detail.run.error ?? "",
+        detail.run.errorCode ?? "",
+        detail.run.stderrExcerpt ?? "",
+        ...detail.events.map((event) => event.message ?? ""),
+    ].join("\n");
+    if (detail.run.status === "timed_out")
+        return "timeout";
+    if (/permission denied/i.test(errorText))
+        return "permission_denied";
+    if (/could not read username|authentication failed|401|403/i.test(errorText))
+        return "auth_or_git_failure";
+    if (/cannot find module|module not found|command not found|no such file/i.test(errorText))
+        return "dependency_or_boot_failure";
+    if (/connection refused|econnrefused|network/i.test(errorText))
+        return "network_dependency_failure";
+    if (/detached|process_lost|orphan/i.test(errorText))
+        return "runtime_process_failure";
+    if (detail.run.status === "failed")
+        return "run_failed_unknown";
+    if (detail.run.status === "cancelled")
+        return "cancelled";
+    return "healthy_or_unknown";
+}
+function computeMetrics(detail) {
+    const { inputTokens: transcriptInputTokens, outputTokens: transcriptOutputTokens, cachedTokens: transcriptCachedTokens } = countTokens(detail.transcript);
+    const usage = detail.run.usageJson ?? {};
+    const inputTokens = Number(usage.inputTokens ?? transcriptInputTokens ?? 0);
+    const outputTokens = Number(usage.outputTokens ?? transcriptOutputTokens ?? 0);
+    const cachedTokens = Number(usage.cachedInputTokens ?? usage.cachedTokens ?? transcriptCachedTokens ?? 0);
+    const costUsd = Number(usage.costUsd ?? usage.totalCostUsd ?? 0);
+    const startedAtMs = detail.run.startedAt ? new Date(detail.run.startedAt).getTime() : null;
+    const createdAtMs = new Date(detail.run.createdAt).getTime();
+    const baselineMs = startedAtMs ?? createdAtMs;
+    let assistantTurns = 0;
+    let toolCalls = 0;
+    let toolResults = 0;
+    let stderrLines = 0;
+    let firstToolCallLatencyMs = null;
+    let firstAssistantOutputLatencyMs = null;
+    const topToolCounts = new Map();
+    for (const entry of detail.transcript) {
+        const tsMs = Number.isFinite(Date.parse(entry.ts)) ? Date.parse(entry.ts) : baselineMs;
+        if (entry.kind === "assistant" || entry.kind === "thinking") {
+            assistantTurns += 1;
+            if (firstAssistantOutputLatencyMs === null) {
+                firstAssistantOutputLatencyMs = Math.max(0, tsMs - baselineMs);
+            }
+        }
+        if (entry.kind === "tool_call") {
+            toolCalls += 1;
+            if (firstToolCallLatencyMs === null) {
+                firstToolCallLatencyMs = Math.max(0, tsMs - baselineMs);
+            }
+            topToolCounts.set(entry.name, (topToolCounts.get(entry.name) ?? 0) + 1);
+        }
+        if (entry.kind === "tool_result") {
+            toolResults += 1;
+        }
+        if (entry.kind === "stderr") {
+            stderrLines += 1;
+        }
+    }
+    const durationMs = detail.run.finishedAt && detail.run.startedAt
+        ? Math.max(0, new Date(detail.run.finishedAt).getTime() - new Date(detail.run.startedAt).getTime())
+        : 0;
+    return {
+        durationMs,
+        inputTokens,
+        outputTokens,
+        cachedTokens,
+        costUsd,
+        assistantTurns,
+        toolCalls,
+        toolResults,
+        stderrLines,
+        firstToolCallLatencyMs,
+        firstAssistantOutputLatencyMs,
+        topTools: [...topToolCounts.entries()]
+            .sort((left, right) => right[1] - left[1])
+            .slice(0, 8)
+            .map(([name, count]) => ({ name, count })),
+    };
+}
+function quickFindings(detail, metrics) {
+    const findings = [];
+    addFinding(findings, detail.run.status === "failed" ? {
+        id: "failed-run",
+        severity: "error",
+        category: "health",
+        title: "Run failed",
+        detail: detail.run.error ?? "The run ended in a failed state.",
+        evidence: [summarizeIssue(detail)],
+    } : null);
+    addFinding(findings, detail.run.status === "running" ? {
+        id: "running-run",
+        severity: "info",
+        category: "health",
+        title: "Run still in progress",
+        detail: "The run is not terminal yet, so findings may still change.",
+        evidence: [summarizeIssue(detail)],
+    } : null);
+    addFinding(findings, metrics.durationMs > 5 * 60 * 1000 ? {
+        id: "long-runtime",
+        severity: "warn",
+        category: "performance",
+        title: "Long runtime",
+        detail: `Run duration exceeded 5 minutes (${Math.round(metrics.durationMs / 1000)}s).`,
+        evidence: [String(metrics.durationMs)],
+    } : null);
+    addFinding(findings, metrics.costUsd > 5 ? {
+        id: "high-cost",
+        severity: "warn",
+        category: "performance",
+        title: "High cost",
+        detail: `Run cost exceeded $5 ($${metrics.costUsd.toFixed(2)}).`,
+        evidence: [`$${metrics.costUsd.toFixed(2)}`],
+    } : null);
+    addFinding(findings, detail.run.error ? {
+        id: "recorded-error",
+        severity: "warn",
+        category: "error",
+        title: "Recorded error",
+        detail: detail.run.error,
+        evidence: [detail.run.errorCode ?? "no-error-code"],
+    } : null);
+    if (findings.length === 0) {
+        findings.push({
+            id: "healthy-run",
+            severity: "info",
+            category: "health",
+            title: "No obvious health warnings",
+            detail: "The run completed without high-signal warning conditions.",
+            evidence: [summarizeIssue(detail)],
+        });
+    }
+    return findings;
+}
+function errorFindings(detail, taxonomy) {
+    const findings = [];
+    const contextLines = detail.events.slice(-5).map((event) => `[${event.seq}] ${event.message ?? event.eventType}`);
+    const evidence = contextLines.length > 0 ? contextLines : [detail.run.error ?? "No event context"];
+    const knownPatterns = [
+        [/could not read username|authentication failed/i, "auth_or_git_failure", "Git or provider authentication is misconfigured."],
+        [/timeout|timed out/i, "timeout", "The run exceeded its allowed execution window."],
+        [/cannot find module|module not found|command not found/i, "dependency_or_boot_failure", "A required dependency or command is missing."],
+        [/permission denied/i, "permission_denied", "The run failed because the runtime lacked required permissions."],
+        [/connection refused|econnrefused/i, "network_dependency_failure", "The run depended on a service that was unavailable."],
+    ];
+    const haystack = [detail.run.error, detail.run.stderrExcerpt, detail.run.stdoutExcerpt].filter(Boolean).join("\n");
+    const matched = knownPatterns.find(([pattern]) => pattern.test(haystack));
+    addFinding(findings, {
+        id: `taxonomy-${taxonomy}`,
+        severity: detail.run.status === "failed" || detail.run.status === "timed_out" ? "error" : "warn",
+        category: "error",
+        title: "Failure taxonomy",
+        detail: taxonomy,
+        evidence,
+    });
+    if (matched) {
+        findings.push({
+            id: `known-pattern-${matched[1]}`,
+            severity: "error",
+            category: "error",
+            title: "Matched known failure signature",
+            detail: matched[2],
+            evidence,
+        });
+    }
+    else if (detail.run.status === "failed" || detail.run.status === "timed_out") {
+        findings.push({
+            id: "unknown-failure-pattern",
+            severity: "warn",
+            category: "error",
+            title: "Unknown failure signature",
+            detail: "The run failed, but no known signature matched. Inspect the raw log and late event stream.",
+            evidence,
+        });
+    }
+    return findings;
+}
+function perfFindings(metrics) {
+    const findings = [];
+    addFinding(findings, metrics.durationMs > 10 * 60 * 1000 ? {
+        id: "duration-over-10m",
+        severity: "warn",
+        category: "performance",
+        title: "Runtime above 10 minutes",
+        detail: `Run duration reached ${Math.round(metrics.durationMs / 1000)} seconds.`,
+        evidence: [String(metrics.durationMs)],
+    } : null);
+    addFinding(findings, metrics.inputTokens > 500_000 ? {
+        id: "very-high-input-tokens",
+        severity: "warn",
+        category: "performance",
+        title: "Very high input token usage",
+        detail: `Input tokens exceeded 500k (${metrics.inputTokens.toLocaleString()}).`,
+        evidence: [String(metrics.inputTokens)],
+    } : null);
+    addFinding(findings, metrics.toolCalls > 100 ? {
+        id: "very-high-tool-volume",
+        severity: "warn",
+        category: "behavior",
+        title: "Very high tool call volume",
+        detail: `Tool calls exceeded 100 (${metrics.toolCalls}).`,
+        evidence: metrics.topTools.map((tool) => `${tool.name}:${tool.count}`),
+    } : null);
+    addFinding(findings, metrics.firstToolCallLatencyMs !== null && metrics.firstToolCallLatencyMs > 30_000 ? {
+        id: "slow-first-tool-call",
+        severity: "warn",
+        category: "performance",
+        title: "Slow first tool call",
+        detail: `The agent took ${Math.round(metrics.firstToolCallLatencyMs / 1000)} seconds before its first tool call.`,
+        evidence: [String(metrics.firstToolCallLatencyMs)],
+    } : null);
+    if (findings.length === 0) {
+        findings.push({
+            id: "no-perf-red-flags",
+            severity: "info",
+            category: "performance",
+            title: "No obvious performance red flags",
+            detail: "The run stayed within the default time, cost, and tool-volume thresholds.",
+            evidence: [],
+        });
+    }
+    return findings;
+}
+function nextStepsFromFindings(findings, taxonomy, detail) {
+    const nextSteps = [];
+    const findingIds = new Set(findings.map((finding) => finding.id));
+    if (taxonomy === "auth_or_git_failure") {
+        nextSteps.push("Verify the runtime's git/provider authentication and rerun after credentials are fixed.");
+    }
+    if (taxonomy === "dependency_or_boot_failure") {
+        nextSteps.push("Check the working directory and required dependencies or CLI commands before retrying.");
+    }
+    if (taxonomy === "timeout") {
+        nextSteps.push("Inspect the late transcript for repeated loops, then decide whether to raise timeout or tighten the prompt.");
+    }
+    if (findingIds.has("very-high-tool-volume")) {
+        nextSteps.push("Inspect bursts of repeated tool calls and batch repeated reads or searches.");
+    }
+    if (findingIds.has("very-high-input-tokens")) {
+        nextSteps.push("Reduce broad context loading and prefer targeted file reads or delta-oriented fetches.");
+    }
+    if (findingIds.has("unknown-failure-pattern")) {
+        nextSteps.push("Open the raw log and final event slice to capture a new failure signature for future taxonomy rules.");
+    }
+    if (detail.transcript.length > 0) {
+        nextSteps.push("Start with a compact trace outline, then expand only the suspicious turn or step instead of loading every payload.");
+    }
+    if (nextSteps.length === 0) {
+        nextSteps.push("Review the transcript and linked issue context to confirm whether the run behavior matched intent.");
+    }
+    return nextSteps;
+}
+export function diagnoseRun(detail, requestedMode = "auto") {
+    const mode = requestedMode === "auto"
+        ? (detail.run.status === "failed" || detail.run.status === "timed_out"
+            ? "full"
+            : "quick")
+        : requestedMode;
+    const metrics = computeMetrics(detail);
+    const taxonomy = inferFailureTaxonomy(detail);
+    const findings = [];
+    for (const finding of quickFindings(detail, metrics))
+        addFinding(findings, finding);
+    if (mode === "error" || mode === "full") {
+        for (const finding of errorFindings(detail, taxonomy))
+            addFinding(findings, finding);
+    }
+    if (mode === "perf" || mode === "full") {
+        for (const finding of perfFindings(metrics))
+            addFinding(findings, finding);
+    }
+    findings.sort((left, right) => {
+        const severityOrder = { error: 0, warn: 1, info: 2 };
+        return severityOrder[left.severity] - severityOrder[right.severity];
+    });
+    const headlineFinding = findings[0];
+    const summary = headlineFinding
+        ? `${headlineFinding.title}: ${headlineFinding.detail}`
+        : `Run ${detail.run.status}`;
+    return {
+        mode,
+        status: detail.run.status,
+        summary,
+        failureTaxonomy: taxonomy,
+        findings,
+        nextSteps: nextStepsFromFindings(findings, taxonomy, detail),
+        metrics,
+    };
+}
+export function compareRunDiagnoses(left, right) {
+    const deltas = [
+        {
+            metric: "status",
+            left: left.status,
+            right: right.status,
+            detail: `Status changed from ${left.status} to ${right.status}.`,
+        },
+        {
+            metric: "durationMs",
+            left: left.metrics.durationMs,
+            right: right.metrics.durationMs,
+            detail: `Duration delta: ${right.metrics.durationMs - left.metrics.durationMs}ms.`,
+        },
+        {
+            metric: "inputTokens",
+            left: left.metrics.inputTokens,
+            right: right.metrics.inputTokens,
+            detail: `Input token delta: ${right.metrics.inputTokens - left.metrics.inputTokens}.`,
+        },
+        {
+            metric: "costUsd",
+            left: left.metrics.costUsd,
+            right: right.metrics.costUsd,
+            detail: `Cost delta: ${(right.metrics.costUsd - left.metrics.costUsd).toFixed(2)} USD.`,
+        },
+        {
+            metric: "toolCalls",
+            left: left.metrics.toolCalls,
+            right: right.metrics.toolCalls,
+            detail: `Tool call delta: ${right.metrics.toolCalls - left.metrics.toolCalls}.`,
+        },
+    ];
+    return {
+        left,
+        right,
+        summary: `Compared ${left.status} vs ${right.status}; top headline moved from "${left.summary}" to "${right.summary}".`,
+        deltas,
+    };
+}
+//# sourceMappingURL=diagnosis.js.map