@ramarivera/coding-agent-langfuse 0.1.53 → 0.1.55

package/README.md

@@ -8,10 +8,12 @@ Langfuse canonical `usage_details` and `cost_details` attributes so historical
 backfills participate in Langfuse model-usage and cost dashboards. Tool calls
 remain child spans under the same session.
 
-Codex `event_msg` `token_count` rows are imported as non-billable accounting
-spans. They are rolling/snapshot telemetry from the Codex session log, not
-individual model generations, so the importer preserves their token details in
-metadata without sending Langfuse generation `usage_details` or `cost_details`.
+Codex `event_msg` `token_count` rows are imported as cost-accounting
+generations when they include input/output/cache buckets for a known priced
+model. The importer preserves the original token snapshot in metadata and sends
+Langfuse canonical `usage_details`/`cost_details` derived from the bucketed
+usage. Total-only Codex snapshots and unknown/free-preview models stay
+non-billable to avoid inventing cost from ambiguous telemetry.
 
 ```sh
 coding-agent-langfuse-backfill --agents codex,claude,grok,pi,opencode
@@ -133,8 +135,8 @@ records a total cost, that recorded value wins. Otherwise, the importer
 calculates per-usage-type USD costs from a model catalog using rates in USD per
 1M tokens.
 
-The built-in catalog covers OpenAI GPT-5.5, GPT-5.4, and GPT-5.3-Codex API list
-pricing, Anthropic Claude Opus/Sonnet 4 API list pricing, plus the toolbox/Pi
+The built-in catalog covers OpenAI GPT-5.5, GPT-5.4, GPT-5.4 Mini, and
+GPT-5.3-Codex API list pricing, Anthropic Claude Opus/Sonnet 4 API list pricing, plus the toolbox/Pi
 models already used in local configuration, including Fireworks Kimi K2.6,
 Fireworks DeepSeek V4 Pro, MiniMax-M3, Together DeepSeek/Kimi/GLM/MiniMax, and
 Zai GLM. `gpt-5.5` is charged at current standard API list price by default:
@@ -145,6 +147,10 @@ writes, `$30.00` 1-hour cache writes, and `$75.00` output per 1M tokens.
 When a billable generation source only records a total token count without
 input/output/cache breakdown, the importer charges that total at the model input
 rate and marks the cost source as `calculated_total_as_input`.
+Codex token-count snapshots are the exception: they are charged only when the
+snapshot has explicit input/output/cache buckets. Codex reasoning tokens are
+kept in metadata, but not charged separately because Codex token-count output
+totals already include reasoning tokens.
 
 Use an override only when you intentionally want a different accounting policy:
 

package/dist/backfill.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import { execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, renameSync, readdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
+import { closeSync, existsSync, mkdirSync, openSync, renameSync, readdirSync, readFileSync, readSync, statSync, writeFileSync, } from "node:fs";
 import { hostname, homedir } from "node:os";
 import { dirname, join } from "node:path";
 const allAgents = ["claude", "codex", "grok", "opencode", "pi"];
 const importStateIdentityVersion = "v9-cost-details";
 const importStateIdentityVersions = {
     claude: "v13-claude-message-snapshot-dedupe",
-    codex: "v11-codex-token-accounting-nonbillable",
+    codex: "v12-codex-token-accounting-priced",
     grok: "v12-cost-details",
     opencode: "v11-cost-details",
     pi: "v12-cost-details",
@@ -24,7 +24,7 @@ const langfuseIdIdentityVersions = {
 const importPayloadVersion = "v10-cost-details";
 const importPayloadVersions = {
     claude: "v11-claude-message-snapshot-dedupe",
-    codex: "v11-codex-token-accounting-nonbillable",
+    codex: "v12-codex-token-accounting-priced",
 };
 const defaultEndpoint = "https://langfuse.ai.roxasroot.net/otel/v1/traces";
 const deadRemoteEndpoint = "http://langfuse.ai.roxasroot.net:14318/v1/traces";
@@ -72,6 +72,12 @@ const gpt54Rates = {
     cacheRead: 0.25,
     cacheWrite: 2.5,
 };
+const gpt54MiniRates = {
+    input: 0.75,
+    output: 4.5,
+    cacheRead: 0.075,
+    cacheWrite: 0.75,
+};
 const gpt53CodexRates = {
     input: 1.75,
     output: 14,
@@ -101,6 +107,8 @@ const defaultCostRates = {
     "openai/gpt-5.5-pro": gpt55ProRates,
     "gpt-5.4": gpt54Rates,
     "openai/gpt-5.4": gpt54Rates,
+    "gpt-5.4-mini": gpt54MiniRates,
+    "openai/gpt-5.4-mini": gpt54MiniRates,
     "gpt-5.3-codex": gpt53CodexRates,
     "openai/gpt-5.3-codex": gpt53CodexRates,
     "claude-opus-4": claudeOpus4Rates,
@@ -761,10 +769,51 @@ function listFiles(root, predicate) {
     return out.sort();
 }
 function parseJsonl(path) {
-    return readFileSync(path, "utf8")
-        .split(/\r?\n/)
-        .filter((line) => line.trim().length > 0)
-        .map((line) => JSON.parse(line));
+    const rows = [];
+    let fd;
+    try {
+        fd = openSync(path, "r");
+        const buffer = Buffer.allocUnsafe(1024 * 1024);
+        let carry = "";
+        let bytesRead = 0;
+        do {
+            bytesRead = readSync(fd, buffer, 0, buffer.length, null);
+            if (bytesRead === 0)
+                break;
+            carry += buffer.subarray(0, bytesRead).toString("utf8");
+            const lines = carry.split(/\r?\n/);
+            carry = lines.pop() ?? "";
+            for (const line of lines)
+                pushJsonlRow(rows, line);
+        } while (bytesRead > 0);
+        pushJsonlRow(rows, carry);
+    }
+    catch {
+        return rows;
+    }
+    finally {
+        if (fd !== undefined) {
+            try {
+                closeSync(fd);
+            }
+            catch {
+                // Ignore close failures; the caller can still use rows parsed so far.
+            }
+        }
+    }
+    return rows;
+}
+function pushJsonlRow(rows, line) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0)
+        return;
+    try {
+        rows.push(JSON.parse(trimmed));
+    }
+    catch {
+        // Session files may be actively written or partially corrupted.
+        // One bad line should not abort the whole collector.
+    }
 }
 function asRecord(value) {
     return value && typeof value === "object"
@@ -949,6 +998,28 @@ function usageTokenTotal(usage) {
             (usage.cacheWrite5m ?? 0) +
             (usage.cacheWrite1h ?? 0);
 }
+function codexBillableTokenUsage(usage) {
+    if (!usage)
+        return undefined;
+    const hasTokenBuckets = [
+        usage.input,
+        usage.output,
+        usage.reasoning,
+        usage.cacheRead,
+        usage.cacheWrite,
+        usage.cacheWrite5m,
+        usage.cacheWrite1h,
+    ].some((value) => value !== undefined && value > 0);
+    if (!hasTokenBuckets)
+        return undefined;
+    return {
+        ...usage,
+        output: usage.output ?? usage.reasoning,
+        // Codex token_count rows report reasoning as a subset of output tokens.
+        // Keep reasoning in metadata, but do not charge it a second time.
+        reasoning: undefined,
+    };
+}
 function textLength(value) {
     if (typeof value === "string")
         return value.length;
@@ -1214,6 +1285,7 @@ function codexEvents(homeDir, options = {}) {
                 const info = asRecord(rowPayload.info);
                 const usage = normalizeUsage(asRecord(info.last_token_usage)) ??
                     normalizeUsage(asRecord(info.total_token_usage));
+                const billableUsage = codexBillableTokenUsage(usage);
                 const tokenKey = JSON.stringify({
                     model: asString(info.model) ?? currentModel,
                     usage,
@@ -1231,9 +1303,10 @@ function codexEvents(homeDir, options = {}) {
                     cwd: currentCwd,
                     startMs: timestamp,
                     parentRecordId: "session",
+                    usage: billableUsage,
                     metadata: {
                         ...pick(info, ["model_context_window"]),
-                        token_usage_billable: false,
+                        token_usage_billable: billableUsage !== undefined,
                         token_usage_source: "codex_event_msg_token_count",
                         token_usage_details: usageDetails(usage),
                     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ramarivera/coding-agent-langfuse",
-  "version": "0.1.53",
+  "version": "0.1.55",
   "description": "Universal coding-agent Langfuse backfiller and live OTLP helpers",
   "type": "module",
   "license": "MIT",