@ramarivera/coding-agent-langfuse 0.1.27 → 0.1.28

package/dist/backfill.d.ts

@@ -40,6 +40,9 @@ type BackfillOptions = {
     sinceMs?: number;
     untilMs?: number;
     batchSize: number;
+    maxRequestBytes: number;
+    maxFieldBytes: number;
+    postDelayMs: number;
 };
 type RunSummary = {
     discovered: Record<string, number>;
@@ -68,7 +71,9 @@ declare function opencodeEvents(homeDir: string, options?: {
     untilMs?: number;
 }): BackfillEvent[];
 declare function fingerprint(event: BackfillEvent): string;
-declare function toOtlp(events: BackfillEvent[]): Record<string, unknown>;
+declare function toOtlp(events: BackfillEvent[], options?: {
+    maxFieldBytes?: number;
+}): Record<string, unknown>;
 declare function discoverEvents(options: BackfillOptions): BackfillEvent[];
 declare function run(options: BackfillOptions): Promise<RunSummary>;
 declare function follow(options: BackfillOptions): Promise<FollowSummary>;

package/dist/backfill.js

@@ -15,6 +15,8 @@ const importIdentityVersions = {
 };
 const defaultEndpoint = "https://langfuse.ai.roxasroot.net/otel/v1/traces";
 const deadRemoteEndpoint = "http://langfuse.ai.roxasroot.net:14318/v1/traces";
+const defaultMaxRequestBytes = 12 * 1024 * 1024;
+const defaultMaxFieldBytes = 512 * 1024;
 const defaultStatePath = join(homedir(), ".local/state/coding-agent-langfuse/backfill-v6.json");
 const currentHost = hostname();
 function usage() {
@@ -29,6 +31,9 @@ Options:
   --until ISO_OR_MS       Only import events before or at this timestamp
   --limit N               Stop after N unsent events
   --batch-size N          OTLP spans per POST (default: 50)
+  --max-request-bytes N   Split OTLP POSTs below this JSON byte size (default: ${defaultMaxRequestBytes})
+  --max-field-bytes N     Truncate individual input/output fields above this byte size (default: ${defaultMaxFieldBytes})
+  --post-delay-ms N       Delay after each successful OTLP POST (default: 0)
   --follow                Keep scanning and sending newly written events
   --poll-interval-ms N    Delay between --follow scans (default: 5000)
   --idle-exit-after-ms N  Stop --follow after this much time without new sends
@@ -45,6 +50,15 @@ function parseArgs(argv) {
     let sinceMs;
     let untilMs;
     let batchSize = 50;
+    let maxRequestBytes = Number.parseInt(process.env.LANGFUSE_BACKFILL_MAX_REQUEST_BYTES ?? "", 10);
+    if (!Number.isFinite(maxRequestBytes))
+        maxRequestBytes = defaultMaxRequestBytes;
+    let maxFieldBytes = Number.parseInt(process.env.LANGFUSE_BACKFILL_MAX_FIELD_BYTES ?? "", 10);
+    if (!Number.isFinite(maxFieldBytes))
+        maxFieldBytes = defaultMaxFieldBytes;
+    let postDelayMs = Number.parseInt(process.env.LANGFUSE_BACKFILL_POST_DELAY_MS ?? "", 10);
+    if (!Number.isFinite(postDelayMs))
+        postDelayMs = 0;
     let follow = false;
     let pollIntervalMs = 5_000;
     let idleExitAfterMs;
@@ -89,6 +103,15 @@ function parseArgs(argv) {
         else if (arg === "--batch-size") {
             batchSize = Number.parseInt(next(), 10);
         }
+        else if (arg === "--max-request-bytes") {
+            maxRequestBytes = Number.parseInt(next(), 10);
+        }
+        else if (arg === "--max-field-bytes") {
+            maxFieldBytes = Number.parseInt(next(), 10);
+        }
+        else if (arg === "--post-delay-ms") {
+            postDelayMs = Number.parseInt(next(), 10);
+        }
         else if (arg === "--follow") {
             follow = true;
         }
@@ -111,6 +134,15 @@ function parseArgs(argv) {
     if (!Number.isFinite(batchSize) || batchSize < 1) {
         throw new Error("--batch-size must be a positive integer");
     }
+    if (!Number.isFinite(maxRequestBytes) || maxRequestBytes < 1024) {
+        throw new Error("--max-request-bytes must be an integer >= 1024");
+    }
+    if (!Number.isFinite(maxFieldBytes) || maxFieldBytes < 1024) {
+        throw new Error("--max-field-bytes must be an integer >= 1024");
+    }
+    if (!Number.isFinite(postDelayMs) || postDelayMs < 0) {
+        throw new Error("--post-delay-ms must be a non-negative integer");
+    }
     if (!Number.isFinite(pollIntervalMs) || pollIntervalMs < 1) {
         throw new Error("--poll-interval-ms must be a positive integer");
     }
@@ -134,6 +166,9 @@ function parseArgs(argv) {
         sinceMs,
         untilMs,
         batchSize,
+        maxRequestBytes,
+        maxFieldBytes,
+        postDelayMs,
     };
 }
 function normalizeEndpoint(endpoint) {
@@ -1124,9 +1159,40 @@ function attr(key, value) {
         return { key, value: { stringValue: value } };
     return { key, value: { stringValue: JSON.stringify(value).slice(0, 8000) } };
 }
-function toOtlp(events) {
+function utf8Bytes(value) {
+    return Buffer.byteLength(value, "utf8");
+}
+function truncateStringByBytes(value, maxBytes) {
+    if (utf8Bytes(value) <= maxBytes)
+        return value;
+    let out = value.slice(0, maxBytes);
+    while (utf8Bytes(out) > maxBytes && out.length > 0) {
+        out = out.slice(0, -1);
+    }
+    return `${out}\n[truncated by coding-agent-langfuse: original_bytes=${utf8Bytes(value)} limit_bytes=${maxBytes}]`;
+}
+function limitValueBytes(value, maxBytes) {
+    if (value === undefined || value === null)
+        return value;
+    if (typeof value === "string")
+        return truncateStringByBytes(value, maxBytes);
+    const json = JSON.stringify(value);
+    if (utf8Bytes(json) <= maxBytes)
+        return value;
+    return truncateStringByBytes(json, maxBytes);
+}
+function limitEventPayload(event, maxFieldBytes) {
+    return {
+        ...event,
+        input: limitValueBytes(event.input, maxFieldBytes),
+        output: limitValueBytes(event.output, maxFieldBytes),
+    };
+}
+function toOtlp(events, options = {}) {
+    const maxFieldBytes = options.maxFieldBytes ?? defaultMaxFieldBytes;
     const spansByTrace = new Map();
-    for (const event of events) {
+    for (const rawEvent of events) {
+        const event = limitEventPayload(rawEvent, maxFieldBytes);
         const id = traceId(event);
         spansByTrace.set(id, [...(spansByTrace.get(id) ?? []), event]);
     }
@@ -1292,13 +1358,55 @@ function toOtlp(events) {
         ],
     };
 }
-async function postOtlp(endpoint, events) {
+function otlpBodyBytes(events, options) {
+    return utf8Bytes(JSON.stringify(toOtlp(events, options)));
+}
+function splitSendBatches(events, options) {
+    const batches = [];
+    let current = [];
+    let currentBytes = 0;
+    const flush = () => {
+        if (current.length === 0)
+            return;
+        batches.push({ events: current, bytes: currentBytes });
+        current = [];
+        currentBytes = 0;
+    };
+    for (const event of events) {
+        const singleBytes = otlpBodyBytes([event], options);
+        if (singleBytes > options.maxRequestBytes) {
+            flush();
+            throw new Error(`Single OTLP event ${event.agent}/${event.sessionId}/${event.recordId} is ${singleBytes} bytes after field truncation, above --max-request-bytes ${options.maxRequestBytes}. Lower --max-field-bytes and rerun.`);
+        }
+        if (current.length === 0) {
+            current = [event];
+            currentBytes = singleBytes;
+            continue;
+        }
+        const nextEvents = [...current, event];
+        const nextBytes = otlpBodyBytes(nextEvents, options);
+        if (current.length >= options.batchSize ||
+            nextBytes > options.maxRequestBytes) {
+            flush();
+            current = [event];
+            currentBytes = singleBytes;
+        }
+        else {
+            current = nextEvents;
+            currentBytes = nextBytes;
+        }
+    }
+    flush();
+    return batches;
+}
+async function postOtlp(endpoint, events, options) {
+    const body = JSON.stringify(toOtlp(events, options));
     let response;
     try {
         response = await fetch(endpoint, {
             method: "POST",
             headers: { "content-type": "application/json" },
-            body: JSON.stringify(toOtlp(events)),
+            body,
         });
     }
     catch (error) {
@@ -1361,22 +1469,54 @@ async function run(options) {
     let aborted = false;
     let lastError;
     if (!options.dryRun) {
-        for (let index = 0; index < selected.length; index += options.batchSize) {
-            const batch = selected.slice(index, index + options.batchSize);
+        let batches;
+        try {
+            batches = splitSendBatches(selected, {
+                batchSize: options.batchSize,
+                maxRequestBytes: options.maxRequestBytes,
+                maxFieldBytes: options.maxFieldBytes,
+            });
+        }
+        catch (error) {
+            aborted = true;
+            lastError = describeError(error);
+            failed = selected.length;
+            console.error(lastError);
+            return {
+                discovered,
+                sent,
+                skipped: events.length - selected.length,
+                failed,
+                notAttempted: 0,
+                aborted,
+                error: lastError,
+                dryRun: options.dryRun,
+                endpoint: options.endpoint,
+                statePath: options.statePath,
+            };
+        }
+        for (let index = 0; index < batches.length; index += 1) {
+            const batch = batches[index]?.events ?? [];
             try {
-                await postOtlp(options.endpoint, batch);
+                await postOtlp(options.endpoint, batch, {
+                    maxFieldBytes: options.maxFieldBytes,
+                });
                 for (const event of batch) {
                     state.sent[fingerprint(event)] = new Date().toISOString();
                 }
                 saveState(options.statePath, state);
                 sent += batch.length;
+                if (options.postDelayMs > 0)
+                    await sleep(options.postDelayMs);
             }
             catch (error) {
                 failed += batch.length;
                 aborted = true;
                 lastError = describeError(error);
-                console.error(`OTLP POST failed for batch ${Math.floor(index / options.batchSize) + 1} (${batch.length} event(s)) to ${options.endpoint}: ${lastError}`);
-                const remaining = selected.length - index - batch.length;
+                console.error(`OTLP POST failed for batch ${index + 1} (${batch.length} event(s)) to ${options.endpoint}: ${lastError}`);
+                const remaining = batches
+                    .slice(index + 1)
+                    .reduce((sum, item) => sum + item.events.length, 0);
                 if (remaining > 0) {
                     console.error(`Aborting backfill after first failed POST; ${remaining} selected event(s) were not attempted. Fix the endpoint and rerun to resume from the local dedupe state.`);
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ramarivera/coding-agent-langfuse",
-  "version": "0.1.27",
+  "version": "0.1.28",
   "description": "Universal coding-agent Langfuse backfiller and live OTLP helpers",
   "type": "module",
   "license": "MIT",