@oxygen-agent/cli 1.1003.12 → 1.1010.644

package/node_modules/@oxygen/shared/dist/langfuse.js

@@ -36,6 +36,8 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";
 import { OXYGEN_VERSION } from "./version.js";
 import { prepareLlmPayload } from "./llm-payload.js";
 import { log } from "./log.js";
+import { llmPromptLabel, registeredLlmPrompt } from "./llm-prompts.js";
+import { activeTelemetryTraceId } from "./telemetry.js";
 const FLUSH_TIMEOUT_MS = 5_000;
 const WARN_THROTTLE_MS = 30_000;
 /**
@@ -99,6 +101,15 @@ export function resolveLlmTracingEnvironment(env = process.env) {
  * the web app nor the worker.
  */
 export function resolveLlmTracingServiceName(env = process.env) {
+    const deployEnv = env.OXYGEN_DEPLOY_ENV?.trim().toLowerCase();
+    if (deployEnv === "selfhost-dev" || deployEnv === "selfhost-prod" || deployEnv === "selfhost-shadow") {
+        if (env.OXYGEN_PROCESS_ROLE === "worker") {
+            return deployEnv === "selfhost-prod" ? "oxygen-worker" : "oxygen-worker-dev";
+        }
+        if (env.OXYGEN_PROCESS_ROLE === "web" || env.OXYGEN_SERVICE_NAME === "oxygen-web")
+            return "oxygen-web";
+        return "oxygen-local";
+    }
     const flyApp = env.FLY_APP_NAME?.trim() || env.FLY_APP?.trim();
     if (flyApp)
         return flyApp.includes("-dev") ? "oxygen-worker-dev" : "oxygen-worker";
@@ -128,6 +139,9 @@ export function llmTraceIdForSeed(seed) {
 function rootSpanIdForSeed(seed) {
     return createHash("sha256").update(`langfuse-root:${seed}`, "utf8").digest("hex").slice(0, 16);
 }
+function observationSpanId(parts) {
+    return createHash("sha256").update(JSON.stringify(parts), "utf8").digest("hex").slice(0, 16);
+}
 // A score's `comment` is free text (a thumbs-down reason is user-authored and
 // unbounded) and the API types it as a STRING, so it cannot take
 // a recoverable payload envelope. It gets the same
@@ -148,9 +162,57 @@ function compact(body) {
     }
     return out;
 }
+// --- Export summary ----------------------------------------------------------
+// ingest_failed is throttled per process, so it is a floor on failures and says
+// nothing about the healthy volume beside it. This rollup gives the ratio: how
+// many spans Langfuse acknowledged, how many batches it refused, and how many
+// observations never reached the exporter at all. Counts only; never content.
+const EXPORT_SUMMARY_INTERVAL_MS = 60_000;
+export function createLlmExportSummary(now = Date.now) {
+    let exported = 0;
+    let failed = 0;
+    let dropped = 0;
+    let windowStartedAt = now();
+    const report = () => {
+        const at = now();
+        if (exported + failed + dropped > 0) {
+            log("info", "llm_tracing.export_summary", {
+                provider: "langfuse",
+                exported,
+                failed,
+                dropped,
+                window_seconds: Math.max(0, Math.round((at - windowStartedAt) / 1000)),
+            });
+        }
+        exported = 0;
+        failed = 0;
+        dropped = 0;
+        windowStartedAt = at;
+    };
+    const add = (count) => (Number.isFinite(count) && count > 0 ? Math.floor(count) : 0);
+    return {
+        exported: (count) => { exported += add(count); },
+        failed: (count) => { failed += add(count); },
+        dropped: (count) => { dropped += add(count); },
+        maybeReport: () => { if (now() - windowStartedAt >= EXPORT_SUMMARY_INTERVAL_MS)
+            report(); },
+        reportNow: report,
+    };
+}
+/**
+ * A thrown value's class name is the only part of it that may leave the
+ * private store: messages from payload preparation can quote the payload.
+ */
+function sanitizedErrorReason(error) {
+    const name = error instanceof Error ? error.name : typeof error;
+    return /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name) ? name : "unknown";
+}
 function boundedNever(rejectable, warn, timeoutMs = FLUSH_TIMEOUT_MS) {
     return new Promise((resolve) => {
-        const timer = setTimeout(resolve, timeoutMs);
+        const timer = setTimeout(() => {
+            warn(new Error("langfuse_flush_deadline_exceeded"));
+            resolve();
+        }, timeoutMs);
         timer.unref?.();
         rejectable
             .catch((error) => warn(error))
@@ -161,17 +223,133 @@ function boundedNever(rejectable, warn, timeoutMs = FLUSH_TIMEOUT_MS) {
     });
 }
 /** Lazy, private, always-sampled OTel transport. Never register it globally. */
-function createOtelEmitter(env, warn) {
+function createOtelEmitter(env, warn, summary) {
     let handle = null;
     const init = () => {
         handle ??= (async () => {
             try {
-                const [{ LangfuseSpanProcessor }, { BasicTracerProvider, AlwaysOnSampler }, { resourceFromAttributes }, tracing, otel, core] = await Promise.all([
+                const [{ LangfuseSpanProcessor }, { BasicTracerProvider, AlwaysOnSampler }, { resourceFromAttributes }, tracing, otel, core, { JsonTraceSerializer }] = await Promise.all([
                     import("@langfuse/otel"), import("@opentelemetry/sdk-trace-base"),
                     import("@opentelemetry/resources"),
                     import("@langfuse/tracing"), import("@opentelemetry/api"), import("@langfuse/core"),
+                    import("@opentelemetry/otlp-transformer"),
                 ]);
+                const exportUrl = `${(env.LANGFUSE_BASE_URL?.trim() || "https://cloud.langfuse.com").replace(/\/+$/, "")}/api/public/otel/v1/traces`;
+                const exportHeaders = {
+                    "Content-Type": "application/json",
+                    Authorization: `Basic ${Buffer.from(`${env.LANGFUSE_PUBLIC_KEY}:${env.LANGFUSE_SECRET_KEY}`).toString("base64")}`,
+                    "x-langfuse-sdk-name": "javascript",
+                    "x-langfuse-sdk-version": core.LANGFUSE_SDK_VERSION,
+                    "x-langfuse-public-key": env.LANGFUSE_PUBLIC_KEY,
+                };
+                const pendingExports = new Set();
+                // The upstream HTTP exporter logs raw partial-success/malformed response
+                // bodies through GLOBAL diagnostics before its callback runs. Keep the
+                // supported OTLP JSON serializer, but own delivery and response handling
+                // inside this private provider. Never mutate the global diagnostic logger.
+                const send = async (spans) => {
+                    const encoded = JsonTraceSerializer.serializeRequest(spans);
+                    if (!encoded)
+                        throw new Error("serialization_failed");
+                    const body = new TextDecoder().decode(encoded);
+                    const signal = AbortSignal.timeout(LANGFUSE_EXPORT_TIMEOUT_SECONDS * 1000);
+                    let response;
+                    for (let attempt = 0; attempt < 3; attempt++) {
+                        try {
+                            response = await fetch(exportUrl, { method: "POST", headers: exportHeaders, body, signal, redirect: "error" });
+                        }
+                        catch {
+                            if (signal.aborted)
+                                throw new Error("deadline_exceeded");
+                            if (attempt === 2)
+                                throw new Error("network_failed");
+                            await new Promise((resolve) => setTimeout(resolve, 100 * (attempt + 1)));
+                            continue;
+                        }
+                        if (![429, 502, 503, 504].includes(response.status) || attempt === 2)
+                            break;
+                        // A retryable rejection has no useful private response to retain.
+                        await response.body?.cancel().catch(() => { });
+                        await new Promise((resolve) => setTimeout(resolve, 100 * (attempt + 1)));
+                    }
+                    if (!response?.ok) {
+                        await response?.body?.cancel().catch(() => { });
+                        throw new Error("http_rejected");
+                    }
+                    const chunks = [];
+                    let bytes = 0;
+                    const reader = response.body?.getReader();
+                    if (reader) {
+                        try {
+                            for (;;) {
+                                const next = await reader.read();
+                                if (next.done)
+                                    break;
+                                bytes += next.value.byteLength;
+                                // OTLP acknowledgements are tiny; an echoed payload must never
+                                // become an unbounded allocation or reach any diagnostic logger.
+                                if (bytes > 64 * 1024)
+                                    throw new Error("response_too_large");
+                                chunks.push(next.value);
+                            }
+                        }
+                        catch (error) {
+                            await reader.cancel().catch(() => { });
+                            throw new Error(error instanceof Error && error.message === "response_too_large" ? "response_too_large" : "response_read_failed");
+                        }
+                        finally {
+                            reader.releaseLock();
+                        }
+                    }
+                    let acknowledgement;
+                    try {
+                        acknowledgement = bytes ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : {};
+                    }
+                    catch {
+                        throw new Error("response_invalid");
+                    }
+                    if (!acknowledgement || typeof acknowledgement !== "object" || Array.isArray(acknowledgement))
+                        throw new Error("response_invalid");
+                    const envelope = acknowledgement;
+                    const partial = envelope.partialSuccess ?? envelope.partial_success;
+                    if (partial !== undefined && partial !== null) {
+                        if (typeof partial !== "object" || Array.isArray(partial))
+                            throw new Error("response_invalid");
+                        const rawRejected = partial.rejectedSpans ?? partial.rejected_spans ?? 0;
+                        if ((typeof rawRejected !== "number" && typeof rawRejected !== "string") || !/^\d+$/.test(String(rawRejected)))
+                            throw new Error("response_invalid");
+                        if (Number(rawRejected) > 0)
+                            throw new Error("partial_rejection");
+                    }
+                };
+                const privateExporter = {
+                    export: (spans, done) => {
+                        const pending = send(spans).then(() => {
+                            // Complete the batch first: a logging fault must never leave the
+                            // processor waiting out its own export timeout.
+                            done({ code: 0 });
+                            summary.exported(spans.length);
+                            summary.maybeReport();
+                        }, (error) => {
+                            const failures = new Set(["serialization_failed", "deadline_exceeded", "network_failed", "http_rejected", "response_too_large", "response_read_failed", "response_invalid", "partial_rejection"]);
+                            const failureKind = error instanceof Error && failures.has(error.message) ? error.message : "transport_failed";
+                            // BSP forwards failures to the global OTel error handler. It only
+                            // receives our constant classification, never the server body.
+                            done({ code: 1, error: new Error("langfuse_delivery_failed") });
+                            summary.failed(spans.length);
+                            summary.maybeReport();
+                            warn(error, { stage: "export", observation_count: spans.length, failure_kind: failureKind });
+                        });
+                        pendingExports.add(pending);
+                        // Even an unexpected SDK callback failure must not create a rejected
+                        // fire-and-forget promise in the product process.
+                        void pending.then(() => pendingExports.delete(pending), () => pendingExports.delete(pending));
+                    },
+                    shutdown: async () => { await Promise.allSettled([...pendingExports]); },
+                    forceFlush: async () => { await Promise.allSettled([...pendingExports]); },
+                };
                 const processor = new LangfuseSpanProcessor({
+                    exporter: privateExporter,
                     publicKey: env.LANGFUSE_PUBLIC_KEY,
                     secretKey: env.LANGFUSE_SECRET_KEY,
                     ...(env.LANGFUSE_BASE_URL?.trim() ? { baseUrl: env.LANGFUSE_BASE_URL.trim() } : {}),
@@ -267,7 +445,19 @@ function createOtelEmitter(env, warn) {
         return handle;
     };
     return {
-        emit: (emission) => { void init().then((h) => h?.emit(emission)).catch((error) => warn(error, { stage: emission.kind })); },
+        emit: (emission) => {
+            void init().then((h) => {
+                // init already warned once; every later emission is lost the same way.
+                // Report from here too: a process whose transport never came up has
+                // no exports to carry the window, and that is when dropped matters.
+                if (!h) {
+                    summary.dropped(1);
+                    summary.maybeReport();
+                    return;
+                }
+                h.emit(emission);
+            }).catch((error) => { summary.dropped(1); summary.maybeReport(); warn(error, { stage: emission.kind }); });
+        },
         flush: () => init().then((h) => h?.processor.forceFlush() ?? Promise.resolve()),
         shutdown: () => init().then((h) => h?.processor.shutdown() ?? Promise.resolve()),
     };
@@ -347,6 +537,126 @@ function createApiScorer(env, warn) {
         },
     };
 }
+const NOOP_PROMPT_RESOLVER = { versionFor: () => undefined };
+const PROMPT_FOUND_TTL_MS = 24 * 60 * 60 * 1000;
+const PROMPT_MISSING_TTL_MS = 10 * 60 * 1000;
+const PROMPT_ERROR_TTL_MS = 60 * 1000;
+const PROMPT_CACHE_LIMIT = 512;
+function promptMetadata(prompt) {
+    return prompt
+        ? { oxygen_prompt_name: prompt.name, oxygen_prompt_hash: prompt.hash, oxygen_prompt_registered: prompt.registered }
+        : {};
+}
+/**
+ * Only a deployed dev process talking to the self-hosted instance may create
+ * prompt versions. Production and shadow data never do (the environment label
+ * is "production" there, or the host is Langfuse Cloud), and a local process
+ * would publish whatever uncommitted template its checkout holds.
+ */
+function mayPublishMissingPrompts(env, baseUrl) {
+    if (resolveLlmTracingEnvironment(env) !== "development")
+        return false;
+    if (resolveLlmTracingServiceName(env) === "oxygen-local")
+        return false;
+    if (env.OXYGEN_DEPLOY_ENV?.trim().toLowerCase() !== "selfhost-dev")
+        return false;
+    let host;
+    try {
+        host = new URL(baseUrl).hostname.toLowerCase();
+    }
+    catch {
+        return false;
+    }
+    return host !== "langfuse.com" && !host.endsWith(".langfuse.com");
+}
+export function createApiPromptResolver(env, warn, fetchImpl = fetch, now = Date.now) {
+    const baseUrl = (env.LANGFUSE_BASE_URL?.trim() || "https://cloud.langfuse.com").replace(/\/+$/, "");
+    const authorization = `Basic ${Buffer.from(`${env.LANGFUSE_PUBLIC_KEY}:${env.LANGFUSE_SECRET_KEY}`).toString("base64")}`;
+    const cache = new Map();
+    const inflight = new Set();
+    const publishAttempted = new Set();
+    const publishes = mayPublishMissingPrompts(env, baseUrl);
+    // One attempt per template per process; any failure falls back to metadata only.
+    const publish = async (key, prompt) => {
+        const definition = publishes && !publishAttempted.has(key) ? registeredLlmPrompt(prompt) : undefined;
+        if (!definition)
+            return undefined;
+        publishAttempted.add(key);
+        const label = llmPromptLabel(prompt.hash);
+        try {
+            const response = await fetchImpl(`${baseUrl}/api/public/v2/prompts`, {
+                method: "POST",
+                headers: { Authorization: authorization, "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    type: "text", name: definition.name, prompt: definition.template, labels: [label],
+                    tags: ["oxygen", "code-owned"], commitMessage: `oxygen code template ${prompt.hash} (runtime publish)`,
+                }),
+                signal: AbortSignal.timeout(5_000),
+                redirect: "error",
+            });
+            if (!response.ok) {
+                await response.body?.cancel().catch(() => { });
+                warn(new Error("langfuse_prompt_link_failed"), { stage: "prompt_publish", status: response.status });
+                return undefined;
+            }
+            const body = await response.json();
+            const labels = Array.isArray(body.labels) ? body.labels : [];
+            return typeof body.version === "number" && Number.isInteger(body.version) && labels.includes(label) ? body.version : undefined;
+        }
+        catch {
+            warn(new Error("langfuse_prompt_link_failed"), { stage: "prompt_publish" });
+            return undefined;
+        }
+    };
+    const lookup = async (key, prompt) => {
+        const label = llmPromptLabel(prompt.hash);
+        let entry = { version: undefined, expiresAt: now() + PROMPT_ERROR_TTL_MS };
+        try {
+            const response = await fetchImpl(`${baseUrl}/api/public/v2/prompts/${encodeURIComponent(prompt.name)}?label=${encodeURIComponent(label)}`, { headers: { Authorization: authorization }, signal: AbortSignal.timeout(5_000), redirect: "error" });
+            if (response.status === 404) {
+                await response.body?.cancel().catch(() => { });
+                const version = await publish(key, prompt);
+                entry = version === undefined
+                    ? { version: undefined, expiresAt: now() + PROMPT_MISSING_TTL_MS }
+                    : { version, expiresAt: now() + PROMPT_FOUND_TTL_MS };
+            }
+            else if (!response.ok) {
+                await response.body?.cancel().catch(() => { });
+                warn(new Error("langfuse_prompt_link_failed"), { stage: "prompt_link", status: response.status });
+            }
+            else {
+                const body = await response.json();
+                const labels = Array.isArray(body.labels) ? body.labels : [];
+                if (typeof body.version === "number" && Number.isInteger(body.version) && labels.includes(label)) {
+                    entry = { version: body.version, expiresAt: now() + PROMPT_FOUND_TTL_MS };
+                }
+                else {
+                    entry = { version: undefined, expiresAt: now() + PROMPT_MISSING_TTL_MS };
+                }
+            }
+        }
+        catch {
+            warn(new Error("langfuse_prompt_link_failed"), { stage: "prompt_link" });
+        }
+        finally {
+            if (cache.size >= PROMPT_CACHE_LIMIT)
+                cache.delete(cache.keys().next().value);
+            cache.set(key, entry);
+            inflight.delete(key);
+        }
+    };
+    return {
+        versionFor: (prompt) => {
+            const key = `${prompt.name}\u0000${prompt.hash}`;
+            const cached = cache.get(key);
+            if ((!cached || cached.expiresAt <= now()) && !inflight.has(key)) {
+                inflight.add(key);
+                void lookup(key, prompt);
+            }
+            return cached?.version;
+        },
+    };
+}
 /**
  * Construct a fail-open Langfuse client, or `null` when tracing is disabled or
  * misconfigured. Prefer the process-wide `getLlmTracingClient` in app code;
@@ -355,24 +665,65 @@ function createApiScorer(env, warn) {
 export function createLlmTracingClient(env = process.env, options) {
     if (!options?.emitterImpl && !isLlmTracingEnabled(env))
         return null;
-    let lastWarnAtMs = 0;
+    const lastWarnAtMs = new Map();
     const warn = (error, context) => {
         const nowMs = Date.now();
-        if (nowMs - lastWarnAtMs < WARN_THROTTLE_MS)
+        const errorCode = error instanceof Error && ["langfuse_flush_deadline_exceeded", "langfuse_replay_identity_unavailable", "langfuse_prompt_link_failed", "langfuse_observation_build_failed"].includes(error.message)
+            ? error.message : "langfuse_delivery_failed";
+        // An identity gap must not suppress a later actual delivery failure.
+        const warningKey = `${errorCode}:${context?.stage ?? "unknown"}`;
+        if (nowMs - (lastWarnAtMs.get(warningKey) ?? -Infinity) < WARN_THROTTLE_MS)
             return;
-        lastWarnAtMs = nowMs;
+        lastWarnAtMs.set(warningKey, nowMs);
         log("warn", "llm_tracing.ingest_failed", {
             provider: "langfuse",
-            error_message: error instanceof Error ? error.message : String(error),
+            // Exporter/API error messages can contain the rejected request or server
+            // response. Only this fixed classification may leave the private store.
+            error_code: errorCode,
             ...context,
         });
     };
-    const transport = options?.emitterImpl ?? createOtelEmitter(env, warn);
+    // Opt-in per environment; managed projects retain their existing append
+    // semantics. A real execution is identified by the caller's immutable start
+    // plus logical id, never by the logical id alone (approval/tool ids can recur).
+    const stableIdentity = env.LANGFUSE_OBSERVATION_IDENTITY === "stable-v1";
+    const physicalId = (body, kind) => {
+        if (!stableIdentity)
+            return undefined;
+        // Verified ledger-backed control hooks supply the exact durable dedupe key.
+        // It already defines identity in the authoritative session/run ledger, so
+        // identity hashing needs no additional clock. The durable capture adapter
+        // separately supplies the original ledger timestamp for v3's date key.
+        const durableEventKey = body.metadata?.oxygen_durable_event_key;
+        if (kind === "event" && typeof durableEventKey === "string" && durableEventKey.length > 0) {
+            return observationSpanId(["langfuse-durable-event-v1", body.traceId, body.name, durableEventKey]);
+        }
+        // Approval ids name a durable decision/request across invocation slices.
+        // Other event keys are not universally unique; do not infer that guarantee.
+        if (kind === "event" && /^apr:(requested|decided):.+/.test(body.id)) {
+            return observationSpanId(["langfuse-event-v1", body.traceId, body.name, body.id]);
+        }
+        if (!body.startTime || !Number.isFinite(body.startTime.getTime())) {
+            warn(new Error("langfuse_replay_identity_unavailable"), { stage: "identity", observation_kind: kind });
+            return undefined;
+        }
+        return observationSpanId(["langfuse-observation-v1", body.traceId, kind, body.name, body.id,
+            body.startTime.toISOString(), body.metadata?.attempt_id ?? null, body.metadata?.call_id ?? null]);
+    };
+    const summary = createLlmExportSummary();
+    const transport = options?.emitterImpl ?? createOtelEmitter(env, warn, summary);
     const emitter = {
         ...transport,
         emit: (emission) => {
             const observationId = emission.observationId ?? randomBytes(8).toString("hex");
             const metadata = { ...emission.attributes.metadata, oxygen_capture_version: 2 };
+            // Cross-link to the operational trace (SigNoz/Axiom). Read here, still in
+            // the caller's synchronous context; the private provider never sees it.
+            if (metadata.otel_trace_id === undefined) {
+                const otelTraceId = activeTelemetryTraceId();
+                if (otelTraceId)
+                    metadata.otel_trace_id = otelTraceId;
+            }
             const attributes = { ...emission.attributes, version: OXYGEN_VERSION, metadata };
             const chunkEmissions = [];
             const capture = (value, field, key) => {
@@ -380,12 +731,18 @@ export function createLlmTracingClient(env = process.env, options) {
                 for (const chunk of prepared.chunks) {
                     chunkEmissions.push({
                         kind: "event", traceSeed: emission.traceSeed, name: "llm.payload_chunk", parentObservationId: observationId,
+                        ...(stableIdentity ? { observationId: observationSpanId(["langfuse-payload-v1", observationId, field, chunk.payload_id, chunk.index]) } : {}),
                         attributes: { version: OXYGEN_VERSION, input: chunk, metadata: {
                                 oxygen_capture_version: 2, oxygen_payload_id: chunk.payload_id,
                                 oxygen_payload_chunk_index: chunk.index, oxygen_payload_chunk_count: prepared.chunks.length,
                                 oxygen_observation_id: `${metadata.oxygen_observation_id ?? observationId}:payload:${field}:${chunk.index}`,
                             } },
-                        correlation: emission.correlation, startTime: emission.endTime ?? emission.startTime, endTime: emission.endTime ?? emission.startTime,
+                        // V3's replacing key includes the start date. The root's end can
+                        // move on resume, while unchanged chunks retain their IDs; pin their
+                        // date to the immutable owner start in stable mode as well.
+                        correlation: emission.correlation,
+                        startTime: stableIdentity ? emission.startTime : emission.endTime ?? emission.startTime,
+                        endTime: stableIdentity ? emission.startTime : emission.endTime ?? emission.startTime,
                     });
                 }
                 return prepared.value;
@@ -414,12 +771,18 @@ export function createLlmTracingClient(env = process.env, options) {
         },
     };
     const scorer = options?.scorerImpl ?? createApiScorer(env, warn);
+    const promptResolver = options?.promptResolverImpl ?? (options?.emitterImpl ? NOOP_PROMPT_RESOLVER : createApiPromptResolver(env, warn));
     const guarded = (fn, stage) => {
         try {
             fn();
         }
         catch (error) {
-            warn(error, { stage });
+            // Building the observation failed before any transport saw it (payload
+            // preparation, identity hashing, a substituted emitter). That is a lost
+            // capture, not a delivery failure, and must not read as one.
+            summary.dropped(1);
+            summary.maybeReport();
+            warn(new Error("langfuse_observation_build_failed"), { stage, reason: sanitizedErrorReason(error) });
         }
     };
     // propagateAttributes rejects non-string ids and anything over 200 chars, and
@@ -442,11 +805,17 @@ export function createLlmTracingClient(env = process.env, options) {
             tags: input.tags,
         });
     };
+    const promptLink = (prompt) => {
+        if (!prompt?.registered)
+            return {};
+        const version = promptResolver.versionFor(prompt);
+        return version === undefined ? {} : { prompt: { name: prompt.name, version, isFallback: false } };
+    };
     const modelObservation = (body, kind) => guarded(() => {
         emitter.emit({
             kind,
             traceSeed: body.traceId,
-            ...(body.observationId ? { observationId: body.observationId } : {}),
+            observationId: body.observationId ?? physicalId(body, kind),
             name: body.name,
             attributes: compact({
                 input: body.input,
@@ -458,7 +827,8 @@ export function createLlmTracingClient(env = process.env, options) {
                 completionStartTime: body.completionStartTime,
                 usageDetails: body.usageDetails,
                 costDetails: body.costDetails,
-                metadata: { ...(body.metadata ?? {}), oxygen_observation_id: body.id },
+                ...promptLink(body.prompt),
+                metadata: { ...(body.metadata ?? {}), ...promptMetadata(body.prompt), oxygen_observation_id: body.id },
             }),
             // The cost-bearing observation: v5 session cost only rolls up when
             // the generation itself carries the session.
@@ -495,6 +865,7 @@ export function createLlmTracingClient(env = process.env, options) {
                 const sliceStart = typeof body.metadata.slice_started_at === "string" ? new Date(body.metadata.slice_started_at) : startTime;
                 emitter.emit({
                     kind: "span", traceSeed: body.id, name: `${body.name}.slice`,
+                    ...(stableIdentity ? { observationId: observationSpanId(["langfuse-slice-v1", body.id, body.metadata.slice_id, sliceStart.getTime()]) } : {}),
                     attributes: compact({ input: body.input, output: body.output, metadata: { ...body.metadata, oxygen_observation_id: `${body.id}:slice:${body.metadata.slice_id}` } }),
                     correlation: correlate({ sessionId: body.sessionId, userId: body.userId }),
                     startTime: Number.isFinite(sliceStart.getTime()) ? sliceStart : startTime, endTime: body.endTime ?? startTime,
@@ -505,6 +876,7 @@ export function createLlmTracingClient(env = process.env, options) {
             emitter.emit({
                 kind: "span",
                 traceSeed: body.traceId,
+                observationId: physicalId(body, "span"),
                 name: body.name,
                 attributes: compact({
                     input: body.input,
@@ -521,10 +893,22 @@ export function createLlmTracingClient(env = process.env, options) {
         generation: (body) => modelObservation(body, "generation"),
         embedding: (body) => modelObservation(body, "embedding"),
         event: (body) => guarded(() => {
+            const durableKey = body.metadata?.oxygen_durable_event_key;
+            const durableEvent = (typeof durableKey === "string" && durableKey.length > 0) || /^apr:(requested|decided):.+/.test(body.id);
+            if (stableIdentity && durableEvent && (!body.startTime || !Number.isFinite(body.startTime.getTime()))) {
+                // A stable id with "now" as its date creates another V3 observation
+                // after midnight. Missing canonical time is missing capture, not a
+                // reason to fabricate a historical event or duplicate it silently.
+                warn(new Error("langfuse_replay_identity_unavailable"), { stage: "identity", observation_kind: "event" });
+                summary.dropped(1);
+                summary.maybeReport();
+                return;
+            }
             const startTime = body.startTime ?? new Date();
             emitter.emit({
                 kind: "event",
                 traceSeed: body.traceId,
+                observationId: physicalId(body, "event"),
                 name: body.name,
                 attributes: compact({
                     input: body.input,
@@ -551,7 +935,14 @@ export function createLlmTracingClient(env = process.env, options) {
         // enforced here (15s for serverless after()) rather than inside one emitter — an emitter that throws
         // synchronously or rejects must still not escape into product code.
         flush: () => boundedNever((async () => emitter.flush())(), (error) => warn(error, { stage: "flush" }), env.VERCEL ? 15_000 : FLUSH_TIMEOUT_MS),
-        shutdown: () => boundedNever((async () => emitter.shutdown())(), (error) => warn(error, { stage: "shutdown" }), env.VERCEL ? 15_000 : FLUSH_TIMEOUT_MS),
+        shutdown: () => boundedNever(
+        // The final window is reported whatever its age: nothing follows it.
+        (async () => { try {
+            await emitter.shutdown();
+        }
+        finally {
+            summary.reportNow();
+        } })(), (error) => warn(error, { stage: "shutdown" }), env.VERCEL ? 15_000 : FLUSH_TIMEOUT_MS),
     };
 }
 // --- Process-wide singleton (both runtimes construct at most one client) ------
@@ -563,6 +954,8 @@ function cacheKey(env) {
         env.LANGFUSE_SECRET_KEY ?? "",
         env.LANGFUSE_BASE_URL ?? "",
         resolveLlmTracingEnvironment(env),
+        resolveLlmTracingServiceName(env),
+        env.LANGFUSE_OBSERVATION_IDENTITY ?? "",
     ].join("|");
 }
 export function getLlmTracingClient(env = process.env) {

package/node_modules/@oxygen/shared/dist/linkedin-countries.d.ts

@@ -30,3 +30,4 @@ export declare const LINKEDIN_COUNTRIES: ReadonlyArray<readonly [code: string, n
  * honest outcome, and the raw payload is still on the record.
  */
 export declare function resolveLinkedInCountry(value: unknown): string | null;
+export { LINKEDIN_COUNTRY_PRIMARY_TIME_ZONES, linkedInCountryPrimaryTimeZone, } from "./linkedin-country-timezones.js";

package/node_modules/@oxygen/shared/dist/linkedin-countries.js

@@ -357,3 +357,5 @@ export function resolveLinkedInCountry(value) {
     const aliased = COUNTRY_ALIASES[lower];
     return aliased ? BY_CODE.get(aliased) ?? null : null;
 }
+// The account-owner country also picks a LinkedIn sender's default working-hours zone.
+export { LINKEDIN_COUNTRY_PRIMARY_TIME_ZONES, linkedInCountryPrimaryTimeZone, } from "./linkedin-country-timezones.js";

package/node_modules/@oxygen/shared/dist/linkedin-country-timezones.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Primary IANA time zone for each LinkedIn country code.
+ *
+ * WHY: a LinkedIn sender's working hours and daily-quota rollover run in the
+ * account owner's local time. The only geo signal OXYGEN holds for a new
+ * account is the country the owner chose at connect ("where the account owner
+ * normally uses LinkedIn"), so that country picks the zone whenever no explicit
+ * timezone was stored. Without it every account ran on America/New_York and a
+ * German account was active 13:00–04:00 Berlin time.
+ *
+ * GENERATED once from ICU's region → zone data (`new Intl.Locale("und-XX")
+ * .timeZones`) for the codes in LINKEDIN_COUNTRIES, then frozen here so a Node
+ * upgrade cannot silently move an account's window. Countries spanning several
+ * zones carry the zone of their largest business population (US → New York,
+ * CA → Toronto, AU → Sydney, BR → São Paulo, RU → Moscow, …). Territories with
+ * no populated zone (AQ, BV, HM) are absent and fall back to the default. An
+ * explicit per-account timezone always wins over this table.
+ */
+export declare const LINKEDIN_COUNTRY_PRIMARY_TIME_ZONES: Readonly<Record<string, string>>;
+/**
+ * The primary IANA zone for an ISO 3166-1 alpha-2 country code (any case), or
+ * null when the code is unknown or has no populated zone.
+ */
+export declare function linkedInCountryPrimaryTimeZone(country: unknown): string | null;