@ramarivera/coding-agent-langfuse 0.1.44 → 0.1.45

package/README.md

@@ -8,6 +8,11 @@ Langfuse canonical `usage_details` and `cost_details` attributes so historical
 backfills participate in Langfuse model-usage and cost dashboards. Tool calls
 remain child spans under the same session.
 
+Codex `event_msg` `token_count` rows are imported as non-billable accounting
+spans. They are rolling/snapshot telemetry from the Codex session log, not
+individual model generations, so the importer preserves their token details in
+metadata without sending Langfuse generation `usage_details` or `cost_details`.
+
 ```sh
 coding-agent-langfuse-backfill --agents codex,claude,grok,pi,opencode
 ```
@@ -22,6 +27,16 @@ npx @ramarivera/coding-agent-langfuse@latest \
   --batch-size 10
 ```
 
+For a Langfuse endpoint that requires project API keys, pass Basic auth
+credentials as `publicKey:secretKey` or set `LANGFUSE_BACKFILL_AUTH`:
+
+```sh
+npx @ramarivera/coding-agent-langfuse@latest \
+  --agents claude,codex,grok,pi,opencode \
+  --endpoint http://127.0.0.1:3000/api/public/otel/v1/traces \
+  --auth pk-lf-example:sk-lf-example
+```
+
 Run live incremental forwarding without putting inference behind a gateway:
 
 ```sh
@@ -54,9 +69,9 @@ Zai GLM. `gpt-5.5` is charged at current standard API list price by default:
 Pro defaults to `$30.00` input and `$180.00` output per 1M tokens. Claude Opus 4
 models default to `$15.00` input, `$1.50` cache hits, `$18.75` 5-minute cache
 writes, `$30.00` 1-hour cache writes, and `$75.00` output per 1M tokens.
-When a source only records a total token count without input/output/cache
-breakdown, the importer charges that total at the model input rate and marks the
-cost source as `calculated_total_as_input`.
+When a billable generation source only records a total token count without
+input/output/cache breakdown, the importer charges that total at the model input
+rate and marks the cost source as `calculated_total_as_input`.
 
 Use an override only when you intentionally want a different accounting policy:
 
@@ -176,6 +191,7 @@ CLI against a local OTLP collector.
 npm run check
 npm test
 npm run test:e2e
+npm run test:e2e:langfuse
 ```
 
 The e2e suite verifies:
@@ -185,5 +201,8 @@ The e2e suite verifies:
 - Follow mode picking up newly written Codex events
 - One CLI run posting reconstructable traces for Claude Code, Codex, Grok,
   OpenCode, and Pi
+- A Docker-backed mini Langfuse import that queries the public observations API
+  and verifies usage/cost fields for sanitized multi-agent sessions, including
+  Claude Code Opus 4.7 and Opus 4.8 cache accounting
 - Service plan generation for Linux systemd user units, macOS LaunchAgents, and
   Windows Scheduled Tasks

package/dist/backfill.d.ts

@@ -33,6 +33,7 @@ type BackfillEvent = {
 type BackfillOptions = {
     agents: Set<AgentName>;
     endpoint: string;
+    auth?: string;
     statePath: string;
     homeDir: string;
     dryRun: boolean;

package/dist/backfill.js

@@ -8,7 +8,7 @@ const allAgents = ["claude", "codex", "grok", "opencode", "pi"];
 const importStateIdentityVersion = "v9-cost-details";
 const importStateIdentityVersions = {
     claude: "v12-cost-details",
-    codex: "v10-cost-details",
+    codex: "v11-codex-token-accounting-nonbillable",
     grok: "v12-cost-details",
     opencode: "v11-cost-details",
     pi: "v12-cost-details",
@@ -22,6 +22,9 @@ const langfuseIdIdentityVersions = {
     pi: "v11-tool-results",
 };
 const importPayloadVersion = "v10-cost-details";
+const importPayloadVersions = {
+    codex: "v11-codex-token-accounting-nonbillable",
+};
 const defaultEndpoint = "https://langfuse.ai.roxasroot.net/otel/v1/traces";
 const deadRemoteEndpoint = "http://langfuse.ai.roxasroot.net:14318/v1/traces";
 const defaultMaxRequestBytes = 12 * 1024 * 1024;
@@ -193,6 +196,7 @@ function usage() {
 
 Options:
   --endpoint URL          OTLP HTTP traces endpoint (default: ${defaultEndpoint})
+  --auth USER:PASS        Optional Langfuse Basic auth credentials
   --agents LIST           Comma-separated agents: claude,codex,grok,opencode,pi
   --state PATH            Dedupe state file (default: ${defaultStatePath})
   --home PATH             Home directory to scan (default: current user home)
@@ -216,6 +220,8 @@ Options:
 }
 function parseArgs(argv) {
     let endpoint = normalizeEndpoint(process.env.LANGFUSE_BACKFILL_ENDPOINT ?? defaultEndpoint);
+    let auth = process.env.LANGFUSE_BACKFILL_AUTH ??
+        process.env.CODING_AGENT_LANGFUSE_AUTH;
     let statePath = process.env.LANGFUSE_BACKFILL_STATE ?? defaultStatePath;
     let homeDir = process.env.HOME ?? homedir();
     let dryRun = false;
@@ -260,6 +266,9 @@ function parseArgs(argv) {
         else if (arg === "--endpoint") {
             endpoint = normalizeEndpoint(next());
         }
+        else if (arg === "--auth") {
+            auth = next();
+        }
         else if (arg === "--state") {
             statePath = next();
         }
@@ -345,9 +354,13 @@ function parseArgs(argv) {
     if (limit !== undefined && (!Number.isFinite(limit) || limit < 1)) {
         throw new Error("--limit must be a positive integer");
     }
+    if (auth !== undefined && auth.trim().length === 0) {
+        throw new Error("--auth must not be empty");
+    }
     return {
         agents,
         endpoint,
+        auth,
         statePath,
         homeDir,
         dryRun,
@@ -952,8 +965,12 @@ function codexEvents(homeDir, options = {}) {
                     cwd: currentCwd,
                     startMs: timestamp,
                     parentRecordId: "session",
-                    usage,
-                    metadata: pick(info, ["model_context_window"]),
+                    metadata: {
+                        ...pick(info, ["model_context_window"]),
+                        token_usage_billable: false,
+                        token_usage_source: "codex_event_msg_token_count",
+                        token_usage_details: usageDetails(usage),
+                    },
                 });
             }
         }
@@ -1492,6 +1509,9 @@ function stableId(input) {
 function importIdentity(event) {
     return importStateIdentityVersions[event.agent] ?? importStateIdentityVersion;
 }
+function payloadVersion(event) {
+    return importPayloadVersions[event.agent] ?? importPayloadVersion;
+}
 function langfuseIdIdentity(event) {
     return langfuseIdIdentityVersions[event.agent] ?? langfuseIdIdentityVersion;
 }
@@ -1599,7 +1619,7 @@ function toOtlp(events, options = {}) {
             attr("langfuse.trace.metadata.project_path", firstProject.projectPath),
             attr("langfuse.trace.metadata.project_name", firstProject.projectName),
             attr("langfuse.trace.metadata.project_folder", firstProject.projectFolder),
-            attr("langfuse.trace.metadata.import_payload_version", importPayloadVersion),
+            attr("langfuse.trace.metadata.import_payload_version", payloadVersion(first)),
             attr("langfuse.trace.metadata.import_state_identity", importIdentity(first)),
             attr("langfuse.trace.metadata.langfuse_id_identity", langfuseIdIdentity(first)),
             attr("langfuse.observation.metadata.agent", first.agent),
@@ -1612,7 +1632,7 @@ function toOtlp(events, options = {}) {
             attr("langfuse.observation.metadata.project_path", firstProject.projectPath),
             attr("langfuse.observation.metadata.project_name", firstProject.projectName),
             attr("langfuse.observation.metadata.project_folder", firstProject.projectFolder),
-            attr("langfuse.observation.metadata.import_payload_version", importPayloadVersion),
+            attr("langfuse.observation.metadata.import_payload_version", payloadVersion(first)),
             attr("langfuse.observation.metadata.import_state_identity", importIdentity(first)),
             attr("langfuse.observation.metadata.langfuse_id_identity", langfuseIdIdentity(first)),
             attr("source.path", first.sourcePath),
@@ -1666,7 +1686,7 @@ function toOtlp(events, options = {}) {
                 attr("langfuse.observation.metadata.project_folder", eventProject.projectFolder),
                 attr("langfuse.observation.metadata.model", modelName ?? event.model),
                 attr("langfuse.observation.metadata.provider", event.provider),
-                attr("langfuse.observation.metadata.import_payload_version", importPayloadVersion),
+                attr("langfuse.observation.metadata.import_payload_version", payloadVersion(event)),
                 attr("langfuse.observation.metadata.import_state_identity", importIdentity(event)),
                 attr("langfuse.observation.metadata.langfuse_id_identity", langfuseIdIdentity(event)),
                 attr("langfuse.observation.usage_details", generation ? usage : undefined),
@@ -1677,6 +1697,9 @@ function toOtlp(events, options = {}) {
                 attr("langfuse.observation.metadata.cost_model_key", cost?.modelKey),
                 attr("langfuse.observation.metadata.cost_rates", cost?.rates),
                 attr("langfuse.observation.metadata.recorded_cost", event.usage?.cost),
+                attr("langfuse.observation.metadata.token_usage_billable", event.metadata?.token_usage_billable),
+                attr("langfuse.observation.metadata.token_usage_source", event.metadata?.token_usage_source),
+                attr("langfuse.observation.metadata.token_usage_details", event.metadata?.token_usage_details),
                 attr("langfuse.observation.input", event.input),
                 attr("langfuse.observation.output", event.output),
                 attr("source.path", event.sourcePath),
@@ -1776,11 +1799,16 @@ function splitSendBatches(events, options) {
 }
 async function postOtlp(endpoint, events, options) {
     const body = JSON.stringify(toOtlp(events, options));
+    const headers = {
+        "content-type": "application/json",
+    };
+    if (options.auth)
+        headers.Authorization = authHeader(options.auth);
     let response;
     try {
         response = await fetch(endpoint, {
             method: "POST",
-            headers: { "content-type": "application/json" },
+            headers,
             body,
         });
     }
@@ -1791,6 +1819,11 @@ async function postOtlp(endpoint, events, options) {
         throw new Error(`OTLP POST failed: ${response.status} ${await response.text()}`);
     }
 }
+function authHeader(auth) {
+    if (/^(Basic|Bearer)\s+/i.test(auth))
+        return auth;
+    return `Basic ${Buffer.from(auth, "utf8").toString("base64")}`;
+}
 function describeError(error) {
     if (!(error instanceof Error))
         return String(error);
@@ -1883,6 +1916,7 @@ async function run(options) {
                 await postOtlp(options.endpoint, batch, {
                     maxFieldBytes: options.maxFieldBytes,
                     costRates: options.costRates,
+                    auth: options.auth,
                 });
                 for (const event of batch) {
                     state.sent[fingerprint(event)] = new Date().toISOString();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ramarivera/coding-agent-langfuse",
-  "version": "0.1.44",
+  "version": "0.1.45",
   "description": "Universal coding-agent Langfuse backfiller and live OTLP helpers",
   "type": "module",
   "license": "MIT",
@@ -25,6 +25,7 @@
     "check": "tsc --noEmit",
     "test": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test test/**/*.test.ts",
     "test:e2e": "npm run build && node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test e2e/test/**/*.test.ts",
+    "test:e2e:langfuse": "npm run build && LANGFUSE_DOCKER_E2E=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test e2e/test/langfuse-docker.test.ts",
     "pack:dry-run": "npm pack --dry-run",
     "prepack": "npm run build"
   },