@ramarivera/coding-agent-langfuse 0.1.44 → 0.1.46

package/README.md

@@ -8,6 +8,11 @@ Langfuse canonical `usage_details` and `cost_details` attributes so historical
 backfills participate in Langfuse model-usage and cost dashboards. Tool calls
 remain child spans under the same session.
 
+Codex `event_msg` `token_count` rows are imported as non-billable accounting
+spans. They are rolling/snapshot telemetry from the Codex session log, not
+individual model generations, so the importer preserves their token details in
+metadata without sending Langfuse generation `usage_details` or `cost_details`.
+
 ```sh
 coding-agent-langfuse-backfill --agents codex,claude,grok,pi,opencode
 ```
@@ -22,6 +27,16 @@ npx @ramarivera/coding-agent-langfuse@latest \
   --batch-size 10
 ```
 
+For a Langfuse endpoint that requires project API keys, pass Basic auth
+credentials as `publicKey:secretKey` or set `LANGFUSE_BACKFILL_AUTH`:
+
+```sh
+npx @ramarivera/coding-agent-langfuse@latest \
+  --agents claude,codex,grok,pi,opencode \
+  --endpoint http://127.0.0.1:3000/api/public/otel/v1/traces \
+  --auth pk-lf-example:sk-lf-example
+```
+
 Run live incremental forwarding without putting inference behind a gateway:
 
 ```sh
@@ -54,9 +69,9 @@ Zai GLM. `gpt-5.5` is charged at current standard API list price by default:
 Pro defaults to `$30.00` input and `$180.00` output per 1M tokens. Claude Opus 4
 models default to `$15.00` input, `$1.50` cache hits, `$18.75` 5-minute cache
 writes, `$30.00` 1-hour cache writes, and `$75.00` output per 1M tokens.
-When a source only records a total token count without input/output/cache
-breakdown, the importer charges that total at the model input rate and marks the
-cost source as `calculated_total_as_input`.
+When a billable generation source only records a total token count without
+input/output/cache breakdown, the importer charges that total at the model input
+rate and marks the cost source as `calculated_total_as_input`.
 
 Use an override only when you intentionally want a different accounting policy:
 
@@ -176,6 +191,7 @@ CLI against a local OTLP collector.
 npm run check
 npm test
 npm run test:e2e
+npm run test:e2e:langfuse
 ```
 
 The e2e suite verifies:
@@ -185,5 +201,8 @@ The e2e suite verifies:
 - Follow mode picking up newly written Codex events
 - One CLI run posting reconstructable traces for Claude Code, Codex, Grok,
   OpenCode, and Pi
+- A Docker-backed mini Langfuse import that queries the public observations API
+  and verifies usage/cost fields for sanitized multi-agent sessions, including
+  Claude Code Opus 4.7 and Opus 4.8 cache accounting
 - Service plan generation for Linux systemd user units, macOS LaunchAgents, and
   Windows Scheduled Tasks

package/dist/backfill.d.ts

@@ -33,6 +33,7 @@ type BackfillEvent = {
 type BackfillOptions = {
     agents: Set<AgentName>;
     endpoint: string;
+    auth?: string;
     statePath: string;
     homeDir: string;
     dryRun: boolean;

package/dist/backfill.js

@@ -7,8 +7,8 @@ import { dirname, join } from "node:path";
 const allAgents = ["claude", "codex", "grok", "opencode", "pi"];
 const importStateIdentityVersion = "v9-cost-details";
 const importStateIdentityVersions = {
-    claude: "v12-cost-details",
-    codex: "v10-cost-details",
+    claude: "v13-claude-message-snapshot-dedupe",
+    codex: "v11-codex-token-accounting-nonbillable",
     grok: "v12-cost-details",
     opencode: "v11-cost-details",
     pi: "v12-cost-details",
@@ -22,6 +22,10 @@ const langfuseIdIdentityVersions = {
     pi: "v11-tool-results",
 };
 const importPayloadVersion = "v10-cost-details";
+const importPayloadVersions = {
+    claude: "v11-claude-message-snapshot-dedupe",
+    codex: "v11-codex-token-accounting-nonbillable",
+};
 const defaultEndpoint = "https://langfuse.ai.roxasroot.net/otel/v1/traces";
 const deadRemoteEndpoint = "http://langfuse.ai.roxasroot.net:14318/v1/traces";
 const defaultMaxRequestBytes = 12 * 1024 * 1024;
@@ -193,6 +197,7 @@ function usage() {
 
 Options:
   --endpoint URL          OTLP HTTP traces endpoint (default: ${defaultEndpoint})
+  --auth USER:PASS        Optional Langfuse Basic auth credentials
   --agents LIST           Comma-separated agents: claude,codex,grok,opencode,pi
   --state PATH            Dedupe state file (default: ${defaultStatePath})
   --home PATH             Home directory to scan (default: current user home)
@@ -216,6 +221,8 @@ Options:
 }
 function parseArgs(argv) {
     let endpoint = normalizeEndpoint(process.env.LANGFUSE_BACKFILL_ENDPOINT ?? defaultEndpoint);
+    let auth = process.env.LANGFUSE_BACKFILL_AUTH ??
+        process.env.CODING_AGENT_LANGFUSE_AUTH;
     let statePath = process.env.LANGFUSE_BACKFILL_STATE ?? defaultStatePath;
     let homeDir = process.env.HOME ?? homedir();
     let dryRun = false;
@@ -260,6 +267,9 @@ function parseArgs(argv) {
         else if (arg === "--endpoint") {
             endpoint = normalizeEndpoint(next());
         }
+        else if (arg === "--auth") {
+            auth = next();
+        }
         else if (arg === "--state") {
             statePath = next();
         }
@@ -345,9 +355,13 @@ function parseArgs(argv) {
     if (limit !== undefined && (!Number.isFinite(limit) || limit < 1)) {
         throw new Error("--limit must be a positive integer");
     }
+    if (auth !== undefined && auth.trim().length === 0) {
+        throw new Error("--auth must not be empty");
+    }
     return {
         agents,
         endpoint,
+        auth,
         statePath,
         homeDir,
         dryRun,
@@ -677,6 +691,25 @@ function usageDetails(usage) {
         details.total = usage.total;
     return Object.keys(details).length > 0 ? details : undefined;
 }
+function usageTokenTotal(usage) {
+    if (!usage)
+        return 0;
+    return usage.total ??
+        (usage.input ?? 0) +
+            (usage.output ?? 0) +
+            (usage.reasoning ?? 0) +
+            (usage.cacheRead ?? 0) +
+            (usage.cacheWrite ?? 0) +
+            (usage.cacheWrite5m ?? 0) +
+            (usage.cacheWrite1h ?? 0);
+}
+function textLength(value) {
+    if (typeof value === "string")
+        return value.length;
+    if (value === undefined || value === null)
+        return 0;
+    return JSON.stringify(value).length;
+}
 function calculateCost(event, usage, costRates) {
     if (!usage)
         return undefined;
@@ -952,8 +985,12 @@ function codexEvents(homeDir, options = {}) {
                     cwd: currentCwd,
                     startMs: timestamp,
                     parentRecordId: "session",
-                    usage,
-                    metadata: pick(info, ["model_context_window"]),
+                    metadata: {
+                        ...pick(info, ["model_context_window"]),
+                        token_usage_billable: false,
+                        token_usage_source: "codex_event_msg_token_count",
+                        token_usage_details: usageDetails(usage),
+                    },
                 });
             }
         }
@@ -1356,6 +1393,7 @@ function genericJsonlEvents(agent, files, sessionName) {
                 startMs,
             },
         ];
+        const claudeAssistantEventsByMessageId = new Map();
         for (const [index, row] of rows.entries()) {
             const message = asRecord(row.message);
             const role = asString(message.role) ?? asString(row.type);
@@ -1363,11 +1401,31 @@ function genericJsonlEvents(agent, files, sessionName) {
                 asString(row.id) ??
                 asString(row.toolUseID) ??
                 `row-${index}`;
+            let childParentRecordId = recordId;
             const toolUseId = asString(row.toolUseID) ?? asString(row.tool_use_id);
             const content = message.content ?? row.content;
             const timestamp = getTimestampMs(row.timestamp ?? row.time_created, startMs + index);
             const usage = normalizeUsage(message.usage ?? row.usage);
-            events.push({
+            const claudeMessageId = agent === "claude" && role === "assistant"
+                ? asString(message.id)
+                : undefined;
+            const eventMetadata = {
+                ...pick(row, [
+                    "type",
+                    "entrypoint",
+                    "version",
+                    "gitBranch",
+                    "error",
+                ]),
+                ...(claudeMessageId
+                    ? {
+                        claude_message_id: claudeMessageId,
+                        claude_snapshot_count: 1,
+                        claude_usage_dedupe: "message_id_max_usage",
+                    }
+                    : {}),
+            };
+            const event = {
                 agent,
                 sourcePath: path,
                 sessionId: asString(row.sessionId) ?? asString(row.session_id) ??
@@ -1391,14 +1449,36 @@ function genericJsonlEvents(agent, files, sessionName) {
                     ? extractText(content)
                     : undefined,
                 usage,
-                metadata: pick(row, [
-                    "type",
-                    "entrypoint",
-                    "version",
-                    "gitBranch",
-                    "error",
-                ]),
-            });
+                metadata: eventMetadata,
+            };
+            if (claudeMessageId) {
+                const existing = claudeAssistantEventsByMessageId.get(claudeMessageId);
+                if (existing) {
+                    childParentRecordId = existing.recordId;
+                    if (usageTokenTotal(event.usage) > usageTokenTotal(existing.usage)) {
+                        existing.usage = event.usage;
+                    }
+                    if (event.output && textLength(event.output) > textLength(existing.output)) {
+                        existing.output = event.output;
+                    }
+                    if (!existing.model && event.model)
+                        existing.model = event.model;
+                    if (!existing.cwd && event.cwd)
+                        existing.cwd = event.cwd;
+                    existing.startMs = Math.min(existing.startMs, event.startMs);
+                    existing.metadata = {
+                        ...existing.metadata,
+                        claude_snapshot_count: (asNumber(existing.metadata?.claude_snapshot_count) ?? 1) + 1,
+                    };
+                }
+                else {
+                    claudeAssistantEventsByMessageId.set(claudeMessageId, event);
+                    events.push(event);
+                }
+            }
+            else {
+                events.push(event);
+            }
             for (const reasoning of reasoningFromContent(content)) {
                 events.push({
                     agent,
@@ -1408,7 +1488,7 @@ function genericJsonlEvents(agent, files, sessionName) {
                     name: `${agent} reasoning`,
                     cwd,
                     startMs: timestamp,
-                    parentRecordId: recordId,
+                    parentRecordId: childParentRecordId,
                     output: reasoning.text,
                     metadata: { has_signature: reasoning.hasSignature },
                 });
@@ -1422,7 +1502,7 @@ function genericJsonlEvents(agent, files, sessionName) {
                     name: `${agent} tool ${tool.name}`,
                     cwd,
                     startMs: timestamp,
-                    parentRecordId: asString(row.uuid) ?? asString(row.id),
+                    parentRecordId: childParentRecordId,
                     input: tool.arguments,
                 });
             }
@@ -1492,6 +1572,9 @@ function stableId(input) {
 function importIdentity(event) {
     return importStateIdentityVersions[event.agent] ?? importStateIdentityVersion;
 }
+function payloadVersion(event) {
+    return importPayloadVersions[event.agent] ?? importPayloadVersion;
+}
 function langfuseIdIdentity(event) {
     return langfuseIdIdentityVersions[event.agent] ?? langfuseIdIdentityVersion;
 }
@@ -1599,7 +1682,7 @@ function toOtlp(events, options = {}) {
             attr("langfuse.trace.metadata.project_path", firstProject.projectPath),
             attr("langfuse.trace.metadata.project_name", firstProject.projectName),
             attr("langfuse.trace.metadata.project_folder", firstProject.projectFolder),
-            attr("langfuse.trace.metadata.import_payload_version", importPayloadVersion),
+            attr("langfuse.trace.metadata.import_payload_version", payloadVersion(first)),
             attr("langfuse.trace.metadata.import_state_identity", importIdentity(first)),
             attr("langfuse.trace.metadata.langfuse_id_identity", langfuseIdIdentity(first)),
             attr("langfuse.observation.metadata.agent", first.agent),
@@ -1612,7 +1695,7 @@ function toOtlp(events, options = {}) {
             attr("langfuse.observation.metadata.project_path", firstProject.projectPath),
             attr("langfuse.observation.metadata.project_name", firstProject.projectName),
             attr("langfuse.observation.metadata.project_folder", firstProject.projectFolder),
-            attr("langfuse.observation.metadata.import_payload_version", importPayloadVersion),
+            attr("langfuse.observation.metadata.import_payload_version", payloadVersion(first)),
             attr("langfuse.observation.metadata.import_state_identity", importIdentity(first)),
             attr("langfuse.observation.metadata.langfuse_id_identity", langfuseIdIdentity(first)),
             attr("source.path", first.sourcePath),
@@ -1666,7 +1749,7 @@ function toOtlp(events, options = {}) {
                 attr("langfuse.observation.metadata.project_folder", eventProject.projectFolder),
                 attr("langfuse.observation.metadata.model", modelName ?? event.model),
                 attr("langfuse.observation.metadata.provider", event.provider),
-                attr("langfuse.observation.metadata.import_payload_version", importPayloadVersion),
+                attr("langfuse.observation.metadata.import_payload_version", payloadVersion(event)),
                 attr("langfuse.observation.metadata.import_state_identity", importIdentity(event)),
                 attr("langfuse.observation.metadata.langfuse_id_identity", langfuseIdIdentity(event)),
                 attr("langfuse.observation.usage_details", generation ? usage : undefined),
@@ -1677,6 +1760,9 @@ function toOtlp(events, options = {}) {
                 attr("langfuse.observation.metadata.cost_model_key", cost?.modelKey),
                 attr("langfuse.observation.metadata.cost_rates", cost?.rates),
                 attr("langfuse.observation.metadata.recorded_cost", event.usage?.cost),
+                attr("langfuse.observation.metadata.token_usage_billable", event.metadata?.token_usage_billable),
+                attr("langfuse.observation.metadata.token_usage_source", event.metadata?.token_usage_source),
+                attr("langfuse.observation.metadata.token_usage_details", event.metadata?.token_usage_details),
                 attr("langfuse.observation.input", event.input),
                 attr("langfuse.observation.output", event.output),
                 attr("source.path", event.sourcePath),
@@ -1776,11 +1862,16 @@ function splitSendBatches(events, options) {
 }
 async function postOtlp(endpoint, events, options) {
     const body = JSON.stringify(toOtlp(events, options));
+    const headers = {
+        "content-type": "application/json",
+    };
+    if (options.auth)
+        headers.Authorization = authHeader(options.auth);
     let response;
     try {
         response = await fetch(endpoint, {
             method: "POST",
-            headers: { "content-type": "application/json" },
+            headers,
             body,
         });
     }
@@ -1791,6 +1882,11 @@ async function postOtlp(endpoint, events, options) {
         throw new Error(`OTLP POST failed: ${response.status} ${await response.text()}`);
     }
 }
+function authHeader(auth) {
+    if (/^(Basic|Bearer)\s+/i.test(auth))
+        return auth;
+    return `Basic ${Buffer.from(auth, "utf8").toString("base64")}`;
+}
 function describeError(error) {
     if (!(error instanceof Error))
         return String(error);
@@ -1883,6 +1979,7 @@ async function run(options) {
                 await postOtlp(options.endpoint, batch, {
                     maxFieldBytes: options.maxFieldBytes,
                     costRates: options.costRates,
+                    auth: options.auth,
                 });
                 for (const event of batch) {
                     state.sent[fingerprint(event)] = new Date().toISOString();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ramarivera/coding-agent-langfuse",
-  "version": "0.1.44",
+  "version": "0.1.46",
   "description": "Universal coding-agent Langfuse backfiller and live OTLP helpers",
   "type": "module",
   "license": "MIT",
@@ -25,6 +25,7 @@
     "check": "tsc --noEmit",
     "test": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test test/**/*.test.ts",
     "test:e2e": "npm run build && node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test e2e/test/**/*.test.ts",
+    "test:e2e:langfuse": "npm run build && LANGFUSE_DOCKER_E2E=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test e2e/test/langfuse-docker.test.ts",
     "pack:dry-run": "npm pack --dry-run",
     "prepack": "npm run build"
   },