@roberttlange/agentlens 0.2.2 → 0.3.0

package/node_modules/@agentlens/core/dist/__tests__/index.test.js

@@ -1,12 +1,49 @@
-import { appendFile, mkdtemp, mkdir, writeFile } from "node:fs/promises";
+import { appendFile, mkdtemp, mkdir, utimes, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import { mergeConfig } from "../config.js";
 import { TraceIndex } from "../traceIndex.js";
 async function createTempRoot() {
     return mkdtemp(path.join(os.tmpdir(), "agentlens-core-"));
 }
+function buildCodexTraceLog(sessionId, sequence) {
+    const firstTs = new Date(Date.UTC(2026, 1, 11, 10, 0, sequence)).toISOString();
+    const secondTs = new Date(Date.UTC(2026, 1, 11, 10, 0, sequence + 1)).toISOString();
+    return [
+        JSON.stringify({
+            timestamp: firstTs,
+            type: "session_meta",
+            payload: { id: sessionId, cwd: "/tmp/project", cli_version: "0.1.0" },
+        }),
+        JSON.stringify({
+            timestamp: secondTs,
+            type: "response_item",
+            payload: {
+                type: "message",
+                role: "assistant",
+                content: [{ type: "output_text", text: `trace ${sequence}` }],
+            },
+        }),
+    ].join("\n");
+}
+async function waitForCondition(predicate, options = {}) {
+    const timeoutMs = options.timeoutMs ?? 5_000;
+    const intervalMs = options.intervalMs ?? 25;
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        try {
+            predicate();
+            return;
+        }
+        catch (error) {
+            if (Date.now() >= deadline) {
+                throw error;
+            }
+            await new Promise((resolve) => setTimeout(resolve, intervalMs));
+        }
+    }
+}
 describe("trace index", () => {
     it("defaults sessionLogDirectories to common agent homes with explicit log types", () => {
         const config = mergeConfig();
@@ -48,6 +85,113 @@ describe("trace index", () => {
             { directory: "~/.pi", logType: "pi" },
         ]);
     });
+    it("arms the watcher before bootstrap hydration during start", async () => {
+        const baseConfig = mergeConfig({ sessionLogDirectories: [] });
+        const config = {
+            ...baseConfig,
+            sessionLogDirectories: [],
+            sources: Object.fromEntries(Object.entries(baseConfig.sources).map(([key, value]) => [
+                key,
+                {
+                    ...value,
+                    enabled: false,
+                    roots: [],
+                },
+            ])),
+        };
+        const index = new TraceIndex(config);
+        const internal = index;
+        const calls = [];
+        const bootstrapSpy = vi.spyOn(internal, "bootstrapRecentTraces").mockImplementation(async () => {
+            calls.push("bootstrap");
+        });
+        const restartWatcherSpy = vi.spyOn(internal, "restartWatcher").mockImplementation(async () => {
+            calls.push("watcher");
+        });
+        try {
+            await index.start();
+            expect(calls).toEqual(["watcher", "bootstrap"]);
+        }
+        finally {
+            bootstrapSpy.mockRestore();
+            restartWatcherSpy.mockRestore();
+            index.stop();
+        }
+    });
+    it("becomes inspector-ready before full hydration finishes during start", async () => {
+        const root = await createTempRoot();
+        const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "11");
+        await mkdir(codexDir, { recursive: true });
+        for (let traceIndex = 0; traceIndex < 130; traceIndex += 1) {
+            const tracePath = path.join(codexDir, `bootstrap-${String(traceIndex).padStart(3, "0")}.jsonl`);
+            await writeFile(tracePath, buildCodexTraceLog(`bootstrap-${traceIndex}`, traceIndex), "utf8");
+            const mtime = new Date(Date.UTC(2026, 1, 11, 10, 0, 0) + (130 - traceIndex) * 1_000);
+            await utimes(tracePath, mtime, mtime);
+        }
+        const config = mergeConfig({
+            sessionLogDirectories: [],
+            sources: {
+                codex_home: {
+                    name: "codex_home",
+                    enabled: true,
+                    roots: [path.join(root, ".codex", "sessions")],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "codex",
+                },
+                claude_projects: {
+                    name: "claude_projects",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+                claude_history: {
+                    name: "claude_history",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["history.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+            },
+        });
+        const index = new TraceIndex(config);
+        try {
+            await index.start();
+            const startup = index.getStartupState();
+            expect(startup.inspectorReady).toBe(true);
+            expect(startup.fullReady).toBe(false);
+            expect(startup.isPartial).toBe(true);
+            expect(startup.discoveredTraceCount).toBe(130);
+            expect(startup.hydratedTraceCount).toBeLessThan(130);
+            expect(index.getSummaries()).toHaveLength(startup.hydratedTraceCount);
+            const allProgress = index.getHydrationProgress(0);
+            expect(allProgress).toEqual({
+                ready: false,
+                relevantDiscoveredCount: 130,
+                relevantHydratedCount: startup.hydratedTraceCount,
+                percent: Math.round((startup.hydratedTraceCount / 130) * 100),
+            });
+            const newestHydratedMtime = Math.min(...index.getSummaries().map((summary) => summary.mtimeMs));
+            const newestWindowProgress = index.getHydrationProgress(newestHydratedMtime);
+            expect(newestWindowProgress.ready).toBe(true);
+            await waitForCondition(() => {
+                const ready = index.getStartupState();
+                expect(ready.fullReady).toBe(true);
+                expect(ready.isPartial).toBe(false);
+                expect(ready.hydratedTraceCount).toBe(130);
+            });
+            expect(index.getSummaries()).toHaveLength(130);
+        }
+        finally {
+            index.stop();
+        }
+    });
     it("parses codex home session files and computes overview", async () => {
         const root = await createTempRoot();
         const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "11");
@@ -1713,6 +1857,135 @@ describe("trace index", () => {
         expect(summary?.costEstimateUsd).not.toBeNull();
         expect((summary?.costEstimateUsd ?? 0) > 0).toBe(true);
     });
+    it("derives timestamped codex usage points from token_count rows", async () => {
+        const root = await createTempRoot();
+        const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "13");
+        await mkdir(codexDir, { recursive: true });
+        const codexPath = path.join(codexDir, "rollout-usage-points.jsonl");
+        await writeFile(codexPath, [
+            JSON.stringify({
+                timestamp: "2026-02-13T23:59:00.000Z",
+                type: "session_meta",
+                payload: { id: "sess-usage-points", cwd: "/tmp/project", cli_version: "0.1.0" },
+            }),
+            JSON.stringify({
+                timestamp: "2026-02-13T23:59:01.000Z",
+                type: "turn_context",
+                payload: { model: "gpt-5.3-codex" },
+            }),
+            JSON.stringify({
+                timestamp: "2026-02-13T23:59:02.000Z",
+                type: "event_msg",
+                payload: {
+                    type: "token_count",
+                    info: {
+                        total_token_usage: {
+                            input_tokens: 100,
+                            cached_input_tokens: 20,
+                            output_tokens: 10,
+                            reasoning_output_tokens: 0,
+                            total_tokens: 130,
+                        },
+                        last_token_usage: {
+                            input_tokens: 100,
+                            cached_input_tokens: 20,
+                            output_tokens: 10,
+                            reasoning_output_tokens: 0,
+                            total_tokens: 130,
+                        },
+                    },
+                },
+            }),
+            JSON.stringify({
+                timestamp: "2026-02-14T00:00:01.000Z",
+                type: "event_msg",
+                payload: {
+                    type: "token_count",
+                    info: {
+                        total_token_usage: {
+                            input_tokens: 140,
+                            cached_input_tokens: 30,
+                            output_tokens: 40,
+                            reasoning_output_tokens: 5,
+                            total_tokens: 215,
+                        },
+                        last_token_usage: {
+                            input_tokens: 40,
+                            cached_input_tokens: 10,
+                            output_tokens: 30,
+                            reasoning_output_tokens: 5,
+                            total_tokens: 85,
+                        },
+                    },
+                },
+            }),
+        ].join("\n"), "utf8");
+        const config = mergeConfig({
+            sessionLogDirectories: [],
+            cost: {
+                enabled: true,
+                currency: "USD",
+                unknownModelPolicy: "n_a",
+                modelRates: [
+                    {
+                        model: "gpt-5.3-codex",
+                        inputPer1MUsd: 1,
+                        outputPer1MUsd: 1,
+                        cachedReadPer1MUsd: 1,
+                        cachedCreatePer1MUsd: 1,
+                        reasoningOutputPer1MUsd: 1,
+                    },
+                ],
+            },
+            sources: {
+                codex_home: {
+                    name: "codex_home",
+                    enabled: true,
+                    roots: [path.join(root, ".codex", "sessions")],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "codex",
+                },
+                claude_projects: {
+                    name: "claude_projects",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+                claude_history: {
+                    name: "claude_history",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["history.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+            },
+        });
+        const index = new TraceIndex(config);
+        await index.refresh();
+        const summary = index.getSummaries()[0];
+        const usageArtifacts = index.getSessionUsageArtifacts(summary.id);
+        expect(usageArtifacts.usagePoints).toHaveLength(2);
+        expect(usageArtifacts.usagePoints[0]).toMatchObject({
+            timestampMs: Date.UTC(2026, 1, 13, 23, 59, 2),
+            inputTokens: 100,
+            cachedReadTokens: 20,
+            outputTokens: 10,
+        });
+        expect(usageArtifacts.usagePoints[1]).toMatchObject({
+            timestampMs: Date.UTC(2026, 1, 14, 0, 0, 1),
+            inputTokens: 40,
+            cachedReadTokens: 10,
+            outputTokens: 30,
+            reasoningOutputTokens: 5,
+        });
+    });
     it("does not double bill codex cached input tokens in cost estimate", async () => {
         const root = await createTempRoot();
         const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "13");
@@ -1806,6 +2079,132 @@ describe("trace index", () => {
         const summary = index.getSummaries()[0];
         expect(summary?.costEstimateUsd).toBe(0.0001);
     });
+    it("applies GPT-5.4 long-context pricing per codex token-count row", async () => {
+        const root = await createTempRoot();
+        const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "13");
+        await mkdir(codexDir, { recursive: true });
+        const codexPath = path.join(codexDir, "rollout-gpt54-tiered.jsonl");
+        await writeFile(codexPath, [
+            JSON.stringify({
+                timestamp: "2026-02-13T12:00:00.000Z",
+                type: "session_meta",
+                payload: { id: "sess-gpt54-tiered", cwd: "/tmp/project", cli_version: "0.1.0" },
+            }),
+            JSON.stringify({
+                timestamp: "2026-02-13T12:00:01.000Z",
+                type: "turn_context",
+                payload: { model: "gpt-5.4-2026-02-28" },
+            }),
+            JSON.stringify({
+                timestamp: "2026-02-13T12:00:02.000Z",
+                type: "event_msg",
+                payload: {
+                    type: "token_count",
+                    info: {
+                        total_token_usage: {
+                            input_tokens: 100_000,
+                            cached_input_tokens: 10_000,
+                            output_tokens: 10_000,
+                            reasoning_output_tokens: 0,
+                            total_tokens: 120_000,
+                        },
+                        last_token_usage: {
+                            input_tokens: 100_000,
+                            cached_input_tokens: 10_000,
+                            output_tokens: 10_000,
+                            reasoning_output_tokens: 0,
+                            total_tokens: 120_000,
+                        },
+                        model_context_window: 1_050_000,
+                    },
+                },
+            }),
+            JSON.stringify({
+                timestamp: "2026-02-13T12:00:03.000Z",
+                type: "event_msg",
+                payload: {
+                    type: "token_count",
+                    info: {
+                        total_token_usage: {
+                            input_tokens: 400_000,
+                            cached_input_tokens: 30_000,
+                            output_tokens: 20_000,
+                            reasoning_output_tokens: 0,
+                            total_tokens: 450_000,
+                        },
+                        last_token_usage: {
+                            input_tokens: 300_000,
+                            cached_input_tokens: 20_000,
+                            output_tokens: 10_000,
+                            reasoning_output_tokens: 0,
+                            total_tokens: 330_000,
+                        },
+                        model_context_window: 1_050_000,
+                    },
+                },
+            }),
+        ].join("\n"), "utf8");
+        const config = mergeConfig({
+            sessionLogDirectories: [],
+            cost: {
+                enabled: true,
+                currency: "USD",
+                unknownModelPolicy: "n_a",
+                modelRates: [
+                    {
+                        model: "gpt-5.4",
+                        inputPer1MUsd: 1.25,
+                        outputPer1MUsd: 7.5,
+                        cachedReadPer1MUsd: 0.125,
+                        cachedCreatePer1MUsd: 0,
+                        reasoningOutputPer1MUsd: 0,
+                        longContextThresholdTokens: 272_000,
+                        longContextInputPer1MUsd: 2.5,
+                        longContextOutputPer1MUsd: 11.25,
+                        contextWindowTokens: 1_050_000,
+                    },
+                ],
+            },
+            models: {
+                defaultContextWindowTokens: 200_000,
+                contextWindows: [{ model: "gpt-5.4", contextWindowTokens: 1_050_000 }],
+            },
+            sources: {
+                codex_home: {
+                    name: "codex_home",
+                    enabled: true,
+                    roots: [path.join(root, ".codex", "sessions")],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "codex",
+                },
+                claude_projects: {
+                    name: "claude_projects",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+                claude_history: {
+                    name: "claude_history",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["history.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+            },
+        });
+        const index = new TraceIndex(config);
+        await index.refresh();
+        const summary = index.getSummaries()[0];
+        expect(summary?.contextWindowPct).toBeCloseTo(30.47619, 5);
+        expect(summary?.costEstimateUsd).toBe(1.06625);
+    });
     it("deduplicates repeated claude usage rows by request id for cost estimation", async () => {
         const root = await createTempRoot();
         const claudeDir = path.join(root, ".claude", "projects", "proj");
@@ -1925,6 +2324,100 @@ describe("trace index", () => {
         const summary = index.getSummaries()[0];
         expect(summary?.costEstimateUsd).toBe(0.001065);
     });
+    it("applies Claude long-context pricing and alias normalization from assistant usage rows", async () => {
+        const root = await createTempRoot();
+        const claudeDir = path.join(root, ".claude", "projects", "proj");
+        await mkdir(claudeDir, { recursive: true });
+        const claudePath = path.join(claudeDir, "session-cost-tiered.jsonl");
+        await writeFile(claudePath, [
+            JSON.stringify({
+                type: "assistant",
+                sessionId: "claude-tiered-sess",
+                uuid: "u1",
+                requestId: "req_tiered_1",
+                message: {
+                    model: "claude-sonnet-4-6-20260219",
+                    id: "msg_1",
+                    type: "message",
+                    role: "assistant",
+                    content: [{ type: "text", text: "tiered" }],
+                    usage: {
+                        input_tokens: 100_000,
+                        cache_creation_input_tokens: 10_000,
+                        cache_read_input_tokens: 120_000,
+                        cache_creation: { ephemeral_5m_input_tokens: 5_000, ephemeral_1h_input_tokens: 5_000 },
+                        output_tokens: 50_000,
+                    },
+                },
+            }),
+        ].join("\n"), "utf8");
+        const config = mergeConfig({
+            sessionLogDirectories: [],
+            cost: {
+                enabled: true,
+                currency: "USD",
+                unknownModelPolicy: "n_a",
+                modelRates: [
+                    {
+                        model: "claude-sonnet-4.6",
+                        inputPer1MUsd: 3,
+                        outputPer1MUsd: 15,
+                        cachedReadPer1MUsd: 0.3,
+                        cachedCreatePer1MUsd: 3.75,
+                        cachedCreate5mPer1MUsd: 3.75,
+                        cachedCreate1hPer1MUsd: 6,
+                        reasoningOutputPer1MUsd: 0,
+                        longContextThresholdTokens: 200_000,
+                        longContextInputPer1MUsd: 6,
+                        longContextOutputPer1MUsd: 22.5,
+                        longContextCachedReadPer1MUsd: 0.6,
+                        longContextCachedCreatePer1MUsd: 7.5,
+                        longContextCachedCreate5mPer1MUsd: 7.5,
+                        longContextCachedCreate1hPer1MUsd: 12,
+                        contextWindowTokens: 1_000_000,
+                    },
+                ],
+            },
+            models: {
+                defaultContextWindowTokens: 200_000,
+                contextWindows: [{ model: "claude-sonnet-4.6", contextWindowTokens: 1_000_000 }],
+            },
+            sources: {
+                claude_projects: {
+                    name: "claude_projects",
+                    enabled: true,
+                    roots: [path.join(root, ".claude", "projects")],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+                codex_home: {
+                    name: "codex_home",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "codex",
+                },
+                claude_history: {
+                    name: "claude_history",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["history.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+            },
+        });
+        const index = new TraceIndex(config);
+        await index.refresh();
+        const summary = index.getSummaries()[0];
+        expect(summary?.contextWindowPct).toBe(23);
+        expect(summary?.costEstimateUsd).toBe(1.8945);
+    });
     it("redacts secret-like values from event previews and raw payloads", async () => {
         const root = await createTempRoot();
         const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "13");
@@ -2108,6 +2601,101 @@ describe("trace index", () => {
             index.stop();
         }
     });
+    it("serves activity artifacts for cold traces without materializing full events", async () => {
+        const root = await createTempRoot();
+        const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "13");
+        await mkdir(codexDir, { recursive: true });
+        for (const [index, filename] of ["rollout-a.jsonl", "rollout-b.jsonl", "rollout-c.jsonl"].entries()) {
+            await writeFile(path.join(codexDir, filename), [
+                JSON.stringify({
+                    timestamp: `2026-02-13T10:00:0${index}.000Z`,
+                    type: "session_meta",
+                    payload: { id: `sess-${index}`, cwd: "/tmp/project", cli_version: "0.1.0" },
+                }),
+                JSON.stringify({
+                    timestamp: `2026-02-13T10:00:1${index}.000Z`,
+                    type: "response_item",
+                    payload: {
+                        type: "message",
+                        role: "assistant",
+                        content: [{ type: "output_text", text: `trace-${index}` }],
+                    },
+                }),
+            ].join("\n") + "\n", "utf8");
+        }
+        const config = mergeConfig({
+            retention: {
+                strategy: "aggressive_recency",
+                hotTraceCount: 1,
+                warmTraceCount: 0,
+                maxResidentEventsPerHotTrace: 1000,
+                maxResidentEventsPerWarmTrace: 50,
+                detailLoadMode: "lazy_from_disk",
+            },
+            sessionLogDirectories: [],
+            sources: {
+                codex_home: {
+                    name: "codex_home",
+                    enabled: true,
+                    roots: [path.join(root, ".codex", "sessions")],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "codex",
+                },
+                claude_projects: {
+                    name: "claude_projects",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["**/*.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+                claude_history: {
+                    name: "claude_history",
+                    enabled: false,
+                    roots: [],
+                    includeGlobs: ["history.jsonl"],
+                    excludeGlobs: [],
+                    maxDepth: 8,
+                    agentHint: "claude",
+                },
+            },
+        });
+        const index = new TraceIndex(config);
+        await index.start();
+        try {
+            const summaries = index.getSummaries();
+            const coldTraceId = summaries[2]?.id;
+            expect(coldTraceId).toBeTruthy();
+            if (!coldTraceId) {
+                throw new Error("missing cold trace");
+            }
+            const internal = index;
+            const entry = internal.entries.get(coldTraceId);
+            expect(entry?.cachedFullEvents).toBeNull();
+            expect(entry?.cachedRawEvents).toBeNull();
+            expect(entry?.activityArtifacts).toMatchObject({
+                eventCount: 2,
+                eventTimestamps: [],
+            });
+            const parseFileSyncSpy = vi.spyOn(internal.parserRegistry, "parseFileSync");
+            const artifacts = index.getSessionActivityArtifacts(coldTraceId);
+            expect(artifacts.eventCount).toBe(2);
+            expect(artifacts.eventTimestamps).toEqual([]);
+            expect(entry?.cachedFullEvents).toBeNull();
+            expect(entry?.cachedRawEvents).toBeNull();
+            expect(entry?.activityArtifacts).toMatchObject({
+                eventCount: 2,
+                eventTimestamps: [],
+            });
+            expect(parseFileSyncSpy).not.toHaveBeenCalled();
+        }
+        finally {
+            index.stop();
+        }
+    });
     it("keeps codex model and cost metrics after appends with strict redaction", async () => {
         const root = await createTempRoot();
         const codexDir = path.join(root, ".codex", "sessions", "2026", "02", "13");