@ramarivera/coding-agent-langfuse 0.1.16 → 0.1.18

package/README.md

@@ -18,7 +18,18 @@ npx @ramarivera/coding-agent-langfuse@latest \
   --agents claude,codex,grok,pi,opencode \
   --endpoint https://langfuse.ai.roxasroot.net/otel/v1/traces \
   --state "$HOME/.local/state/coding-agent-langfuse/backfill-v6.json" \
-  --batch-size 1000
+  --batch-size 10
+```
+
+Run live incremental forwarding without putting inference behind a gateway:
+
+```sh
+npx @ramarivera/coding-agent-langfuse@latest \
+  --agents codex \
+  --endpoint https://langfuse.ai.roxasroot.net/otel/v1/traces \
+  --state "$HOME/.local/state/coding-agent-langfuse/live-codex.json" \
+  --batch-size 10 \
+  --follow
 ```
 
 The importer is fail-fast: the first failed OTLP POST stops the run, prints the

package/dist/backfill.d.ts

@@ -33,6 +33,9 @@ type BackfillOptions = {
     statePath: string;
     homeDir: string;
     dryRun: boolean;
+    follow: boolean;
+    pollIntervalMs: number;
+    idleExitAfterMs?: number;
     limit?: number;
     sinceMs?: number;
     untilMs?: number;
@@ -50,6 +53,10 @@ type RunSummary = {
     endpoint: string;
     statePath: string;
 };
+type FollowSummary = RunSummary & {
+    iterations: number;
+    follow: true;
+};
 declare function parseArgs(argv: string[]): BackfillOptions;
 declare function codexEvents(homeDir: string): BackfillEvent[];
 declare function claudeEvents(homeDir: string): BackfillEvent[];
@@ -60,5 +67,6 @@ declare function fingerprint(event: BackfillEvent): string;
 declare function toOtlp(events: BackfillEvent[]): Record<string, unknown>;
 declare function discoverEvents(options: BackfillOptions): BackfillEvent[];
 declare function run(options: BackfillOptions): Promise<RunSummary>;
-declare function main(argv?: string[]): Promise<RunSummary>;
-export { type BackfillEvent, type BackfillOptions, claudeEvents, codexEvents, discoverEvents, fingerprint, grokEvents, main, opencodeEvents, parseArgs, piEvents, run, toOtlp, };
+declare function follow(options: BackfillOptions): Promise<FollowSummary>;
+declare function main(argv?: string[]): Promise<RunSummary | FollowSummary>;
+export { type BackfillEvent, type BackfillOptions, claudeEvents, codexEvents, discoverEvents, fingerprint, follow, grokEvents, main, opencodeEvents, parseArgs, piEvents, run, toOtlp, };

package/dist/backfill.js

@@ -5,7 +5,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSy
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 const allAgents = ["claude", "codex", "grok", "opencode", "pi"];
-const importIdentityVersion = "v8-cached-input-token-split";
+const importIdentityVersion = "v9-codex-conversation-events";
 const defaultEndpoint = "https://langfuse.ai.roxasroot.net/otel/v1/traces";
 const deadRemoteEndpoint = "http://langfuse.ai.roxasroot.net:14318/v1/traces";
 const defaultStatePath = join(homedir(), ".local/state/coding-agent-langfuse/backfill-v6.json");
@@ -21,6 +21,9 @@ Options:
   --until ISO_OR_MS       Only import events before or at this timestamp
   --limit N               Stop after N unsent events
   --batch-size N          OTLP spans per POST (default: 50)
+  --follow                Keep scanning and sending newly written events
+  --poll-interval-ms N    Delay between --follow scans (default: 5000)
+  --idle-exit-after-ms N  Stop --follow after this much time without new sends
   --dry-run               Discover and dedupe without sending or mutating state
   --help                  Show this help
 `;
@@ -34,6 +37,9 @@ function parseArgs(argv) {
     let sinceMs;
     let untilMs;
     let batchSize = 50;
+    let follow = false;
+    let pollIntervalMs = 5_000;
+    let idleExitAfterMs;
     const agents = new Set(allAgents);
     for (let i = 0; i < argv.length; i++) {
         const arg = argv[i];
@@ -75,6 +81,15 @@ function parseArgs(argv) {
         else if (arg === "--batch-size") {
             batchSize = Number.parseInt(next(), 10);
         }
+        else if (arg === "--follow") {
+            follow = true;
+        }
+        else if (arg === "--poll-interval-ms") {
+            pollIntervalMs = Number.parseInt(next(), 10);
+        }
+        else if (arg === "--idle-exit-after-ms") {
+            idleExitAfterMs = Number.parseInt(next(), 10);
+        }
         else if (arg === "--since") {
             sinceMs = parseTime(next());
         }
@@ -88,6 +103,13 @@ function parseArgs(argv) {
     if (!Number.isFinite(batchSize) || batchSize < 1) {
         throw new Error("--batch-size must be a positive integer");
     }
+    if (!Number.isFinite(pollIntervalMs) || pollIntervalMs < 1) {
+        throw new Error("--poll-interval-ms must be a positive integer");
+    }
+    if (idleExitAfterMs !== undefined &&
+        (!Number.isFinite(idleExitAfterMs) || idleExitAfterMs < 1)) {
+        throw new Error("--idle-exit-after-ms must be a positive integer");
+    }
     if (limit !== undefined && (!Number.isFinite(limit) || limit < 1)) {
         throw new Error("--limit must be a positive integer");
     }
@@ -97,6 +119,9 @@ function parseArgs(argv) {
         statePath,
         homeDir,
         dryRun,
+        follow,
+        pollIntervalMs,
+        idleExitAfterMs,
         limit,
         sinceMs,
         untilMs,
@@ -204,7 +229,10 @@ function extractText(value, maxLength = 4000) {
         const text = value
             .map((item) => {
             const record = asRecord(item);
-            return asString(record.text) ?? asString(record.content) ?? "";
+            return asString(record.text) ??
+                extractText(record.content, maxLength) ??
+                extractText(record.summary, maxLength) ??
+                "";
         })
             .filter(Boolean)
             .join("\n");
@@ -338,7 +366,8 @@ function costDetails(usage, model) {
     return undefined;
 }
 function isGenerationEvent(event) {
-    return event.usage !== undefined && event.role !== "user";
+    return event.usage !== undefined && event.role !== "user" &&
+        event.role !== "developer" && event.role !== "system";
 }
 function codexEvents(homeDir) {
     const files = listFiles(join(homeDir, ".codex/sessions"), (path) => path.endsWith(".jsonl"));
@@ -353,6 +382,7 @@ function codexEvents(homeDir) {
             asString(getPath(payload, ["model"]));
         let currentModel = model;
         let currentCwd = cwd;
+        let currentTurnRecordId = "session";
         const seenTokenCounts = new Set();
         const events = [
             {
@@ -381,11 +411,12 @@ function codexEvents(homeDir) {
             if (type === "turn_context") {
                 currentModel = asString(rowPayload.model) ?? currentModel;
                 currentCwd = asString(rowPayload.cwd) ?? currentCwd;
+                currentTurnRecordId = `turn-${asString(rowPayload.turn_id) ?? index}`;
                 events.push({
                     agent: "codex",
                     sourcePath: path,
                     sessionId,
-                    recordId: `turn-${asString(rowPayload.turn_id) ?? index}`,
+                    recordId: currentTurnRecordId,
                     name: "codex turn",
                     cwd: currentCwd,
                     model: currentModel,
@@ -395,34 +426,99 @@ function codexEvents(homeDir) {
                 });
             }
             if (type === "response_item" && itemType === "message") {
+                const role = asString(rowPayload.role);
+                const text = extractText(rowPayload.content);
+                const input = role === "user" || role === "developer" ||
+                    role === "system"
+                    ? text
+                    : undefined;
+                const output = role === "assistant" ? text : undefined;
                 events.push({
                     agent: "codex",
                     sourcePath: path,
                     sessionId,
                     recordId: `message-${asString(rowPayload.id) ?? index}`,
-                    name: `codex ${asString(rowPayload.role) ?? "message"}`,
-                    role: asString(rowPayload.role),
+                    name: `codex ${role ?? "message"}`,
+                    role,
                     model: currentModel,
                     cwd: currentCwd,
                     startMs: timestamp,
-                    parentRecordId: "session",
-                    output: extractText(rowPayload.content),
+                    parentRecordId: currentTurnRecordId,
+                    input,
+                    output,
                     usage: normalizeUsage(rowPayload.usage),
+                    metadata: pick(rowPayload, ["phase"]),
+                });
+            }
+            if (type === "response_item" && itemType === "reasoning") {
+                events.push({
+                    agent: "codex",
+                    sourcePath: path,
+                    sessionId,
+                    recordId: `reasoning-${index}`,
+                    name: "codex reasoning",
+                    model: currentModel,
+                    cwd: currentCwd,
+                    startMs: timestamp,
+                    parentRecordId: currentTurnRecordId,
+                    output: extractText(rowPayload.content) ??
+                        extractText(rowPayload.summary),
+                    metadata: {
+                        has_encrypted_content: rowPayload.encrypted_content !== undefined,
+                    },
                 });
             }
             if (type === "response_item" && itemType === "function_call") {
+                const callId = asString(rowPayload.call_id) ?? `${index}`;
                 events.push({
                     agent: "codex",
                     sourcePath: path,
                     sessionId,
-                    recordId: `tool-${asString(rowPayload.call_id) ?? index}`,
+                    recordId: `tool-${callId}`,
                     name: `codex tool ${asString(rowPayload.name) ?? "call"}`,
                     model: currentModel,
                     cwd: currentCwd,
                     startMs: timestamp,
-                    parentRecordId: "session",
+                    parentRecordId: currentTurnRecordId,
                     input: rowPayload.arguments,
-                    metadata: pick(rowPayload, ["name", "call_id"]),
+                    metadata: pick(rowPayload, ["name", "namespace", "call_id"]),
+                });
+            }
+            if (type === "response_item" && itemType === "function_call_output") {
+                const callId = asString(rowPayload.call_id) ?? `${index}`;
+                events.push({
+                    agent: "codex",
+                    sourcePath: path,
+                    sessionId,
+                    recordId: `tool-result-${callId}`,
+                    name: "codex tool result",
+                    model: currentModel,
+                    cwd: currentCwd,
+                    startMs: timestamp,
+                    parentRecordId: `tool-${callId}`,
+                    output: rowPayload.output,
+                    metadata: pick(rowPayload, ["call_id", "status", "execution"]),
+                });
+            }
+            if (type === "response_item" &&
+                (itemType === "tool_search_call" || itemType === "tool_search_output")) {
+                const callId = asString(rowPayload.call_id) ?? `${index}`;
+                const isOutput = itemType === "tool_search_output";
+                events.push({
+                    agent: "codex",
+                    sourcePath: path,
+                    sessionId,
+                    recordId: `${isOutput ? "tool-search-result" : "tool-search"}-${callId}`,
+                    name: isOutput ? "codex tool_search result" : "codex tool_search",
+                    model: currentModel,
+                    cwd: currentCwd,
+                    startMs: timestamp,
+                    parentRecordId: isOutput
+                        ? `tool-search-${callId}`
+                        : currentTurnRecordId,
+                    input: isOutput ? undefined : rowPayload.arguments,
+                    output: isOutput ? rowPayload.tools : undefined,
+                    metadata: pick(rowPayload, ["call_id", "status", "execution"]),
                 });
             }
             if (type === "event_msg" && rowPayload.type === "token_count") {
@@ -837,6 +933,7 @@ function toOtlp(events) {
             attributes: rootAttributes,
             status: { code: 1 },
         };
+        const spanIdsByRecordId = new Map(sortedEvents.map((event) => [event.recordId, spanId(event)]));
         const childSpans = sortedEvents.map((event) => {
             const startMs = event.startMs;
             const durationMs = Math.max(1, (event.endMs ?? event.startMs + 1) - event.startMs);
@@ -898,7 +995,10 @@ function toOtlp(events) {
             return {
                 traceId: traceId(event),
                 spanId: spanId(event),
-                parentSpanId: rootSpanId(event),
+                parentSpanId: event.parentRecordId &&
+                    event.parentRecordId !== "session"
+                    ? spanIdsByRecordId.get(event.parentRecordId) ?? rootSpanId(event)
+                    : rootSpanId(event),
                 name: event.name,
                 kind: 1,
                 startTimeUnixNano: ns(startMs),
@@ -1035,8 +1135,63 @@ async function run(options) {
         statePath: options.statePath,
     };
 }
+function mergeSummary(base, next) {
+    return {
+        discovered: Object.fromEntries(allAgents.map((agent) => [
+            agent,
+            Math.max(base.discovered[agent] ?? 0, next.discovered[agent] ?? 0),
+        ])),
+        sent: base.sent + next.sent,
+        skipped: next.skipped,
+        failed: base.failed + next.failed,
+        notAttempted: next.notAttempted,
+        aborted: base.aborted || next.aborted,
+        ...(next.error ? { error: next.error } : base.error ? { error: base.error } : {}),
+        dryRun: base.dryRun,
+        endpoint: base.endpoint,
+        statePath: base.statePath,
+    };
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function follow(options) {
+    let iterations = 0;
+    let lastSendOrStartMs = Date.now();
+    let aggregate;
+    while (true) {
+        iterations += 1;
+        const summary = await run(options);
+        aggregate = aggregate === undefined ? summary : mergeSummary(aggregate, summary);
+        if (summary.sent > 0)
+            lastSendOrStartMs = Date.now();
+        if (summary.aborted)
+            break;
+        if (options.idleExitAfterMs !== undefined &&
+            Date.now() - lastSendOrStartMs >= options.idleExitAfterMs) {
+            break;
+        }
+        await sleep(options.pollIntervalMs);
+    }
+    return {
+        ...(aggregate ?? {
+            discovered: Object.fromEntries(allAgents.map((agent) => [agent, 0])),
+            sent: 0,
+            skipped: 0,
+            failed: 0,
+            notAttempted: 0,
+            aborted: false,
+            dryRun: options.dryRun,
+            endpoint: options.endpoint,
+            statePath: options.statePath,
+        }),
+        iterations,
+        follow: true,
+    };
+}
 async function main(argv = process.argv.slice(2)) {
-    const summary = await run(parseArgs(argv));
+    const options = parseArgs(argv);
+    const summary = options.follow ? await follow(options) : await run(options);
     console.log(JSON.stringify(summary, null, 2));
     return summary;
 }
@@ -1050,4 +1205,4 @@ if (import.meta.url === `file://${process.argv[1]}`) {
         process.exit(1);
     }
 }
-export { claudeEvents, codexEvents, discoverEvents, fingerprint, grokEvents, main, opencodeEvents, parseArgs, piEvents, run, toOtlp, };
+export { claudeEvents, codexEvents, discoverEvents, fingerprint, follow, grokEvents, main, opencodeEvents, parseArgs, piEvents, run, toOtlp, };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ramarivera/coding-agent-langfuse",
-  "version": "0.1.16",
+  "version": "0.1.18",
   "description": "Universal coding-agent Langfuse backfiller and live OTLP helpers",
   "type": "module",
   "license": "MIT",