@smithers-orchestrator/observability 0.24.2 → 0.25.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smithers-orchestrator/observability",
-  "version": "0.24.2",
+  "version": "0.25.1",
   "description": "Concrete Smithers metrics, logging, tracing, and observability integrations",
   "type": "module",
   "sideEffects": false,
@@ -34,8 +34,7 @@
     "@effect/opentelemetry": "^0.63.0",
     "@effect/platform": "^0.96.0",
     "@effect/platform-bun": "^0.89.0",
-    "effect": "^3.21.1",
-    "@smithers-orchestrator/agents": "0.24.2"
+    "effect": "^3.21.1"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/ResolvedSmithersObservabilityOptions.ts

@@ -7,4 +7,5 @@ export type ResolvedSmithersObservabilityOptions = {
   readonly serviceName: string;
   readonly logFormat: SmithersLogFormat;
   readonly logLevel: LogLevel.LogLevel;
+  readonly installLogger: boolean;
 };

package/src/SmithersEvent.ts

@@ -121,7 +121,19 @@ export type SmithersEvent =
       after: RunState;
       timestampMs: number;
     }
-  | { type: "RunFinished"; runId: string; timestampMs: number }
+  | {
+      type: "RunFinished";
+      runId: string;
+      timestampMs: number;
+      /**
+       * Tasks that ended `failed` but were tolerated (continueOnFail / transient
+       * agent failures) so the run still finished. Present only when `> 0` — the
+       * run "succeeded" while these children failed. See `docs/runtime/run-state.mdx`.
+       */
+      failedChildren?: number;
+      /** Task state keys (`nodeId::iteration`) counted by `failedChildren`. */
+      failedChildKeys?: readonly string[];
+    }
   | { type: "RunFailed"; runId: string; error: unknown; timestampMs: number }
   | { type: "RunCancelled"; runId: string; timestampMs: number }
   | {
@@ -229,6 +241,11 @@ export type SmithersEvent =
       runId: string;
       frameNo: number;
       xmlHash: string;
+      trigger?: {
+        reason: string;
+        nodeId?: string;
+        iteration?: number;
+      };
       timestampMs: number;
     }
   | {

package/src/SmithersObservabilityOptions.ts

@@ -7,4 +7,5 @@ export type SmithersObservabilityOptions = {
   readonly serviceName?: string;
   readonly logFormat?: SmithersLogFormat;
   readonly logLevel?: LogLevel.LogLevel | string;
+  readonly installLogger?: boolean;
 };

package/src/_coreCorrelation/withCorrelationContext.js

@@ -5,6 +5,19 @@ import { mergeCorrelationContext } from "./mergeCorrelationContext.js";
 /** @typedef {import("./CorrelationPatch.ts").CorrelationPatch} CorrelationPatch */
 
 /**
+ * Bridge the Effect-tracked correlation context onto the imperative
+ * AsyncLocalStorage store so plain (non-Effect) `getCurrentCorrelationContext()`
+ * reads — e.g. from the imperative logger — see the active run/node correlation
+ * while the effect executes.
+ *
+ * IMPORTANT: run the resulting effect with `Effect.runPromise`/`runFork`, never
+ * `Effect.runSync`. The acquire step calls `AsyncLocalStorage.enterWith()`, which
+ * mutates the *caller's* async context. Under `runSync` the caller is whatever
+ * synchronous context invoked it (e.g. a test-runner's root context); enabling
+ * ALS async-hooks there leaks into every subsequent timer/promise on that
+ * context. `runPromise`/`runFork` execute on an ephemeral fiber context, keeping
+ * the enterWith scoped to that fiber.
+ *
  * @template A, E, R
  * @param {Effect.Effect<A, E, R>} effect
  * @param {CorrelationPatch} patch

package/src/_coreMetrics.js

@@ -1,16 +1,7 @@
-import { Context, Effect, Layer } from "effect";
-import { renderPrometheusSamples, toPrometheusMetricName, } from "./_corePrometheus.js";
-/** @typedef {import("./MetricName.ts").MetricName} MetricName */
+import { Context } from "effect";
+import { toPrometheusMetricName } from "./_corePrometheus.js";
 /** @typedef {import("./SmithersMetricType.ts").SmithersMetricType} SmithersMetricType */
-/** @typedef {import("./SmithersMetricUnit.ts").SmithersMetricUnit} SmithersMetricUnit */
-/** @typedef {import("./_corePrometheusShape.ts").MetricLabels} MetricLabels */
-/** @typedef {import("./_corePrometheusShape.ts").PrometheusSample} PrometheusSample */
-/** @typedef {import("./_coreMetricsShape.ts").CounterEntry} CounterEntry */
-/** @typedef {import("./_coreMetricsShape.ts").GaugeEntry} GaugeEntry */
-/** @typedef {import("./_coreMetricsShape.ts").HistogramEntry} HistogramEntry */
-/** @typedef {import("./_coreMetricsShape.ts").MetricsSnapshot} MetricsSnapshot */
 /** @typedef {import("./_coreMetricsShape.ts").MetricsServiceShape} MetricsServiceShape */
-/** @typedef {import("./_coreMetricsShape.ts").SmithersMetricEvent} SmithersMetricEvent */
 /** @typedef {import("./SmithersMetricDefinition.ts").SmithersMetricDefinition} SmithersMetricDefinition */
 
 /**
@@ -204,307 +195,3 @@ export const smithersMetrics = Object.freeze(Object.fromEntries(smithersMetricCa
 const _MetricsServiceBase = /** @type {Context.TagClass<MetricsService, "MetricsService", MetricsServiceShape>} */ (/** @type {unknown} */ (Context.Tag("MetricsService")()));
 export class MetricsService extends _MetricsServiceBase {
 }
-const DEFAULT_HISTOGRAM_BUCKETS = [
-    1,
-    5,
-    10,
-    25,
-    50,
-    100,
-    250,
-    500,
-    1_000,
-    2_500,
-    5_000,
-    10_000,
-    30_000,
-];
-/**
- * @param {MetricLabels} [labels]
- * @returns {string}
- */
-function labelsKey(labels = {}) {
-    return JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
-}
-/**
- * @param {string} name
- * @param {MetricLabels} [labels]
- * @returns {string}
- */
-function metricKey(name, labels) {
-    return `${name}|${labelsKey(labels)}`;
-}
-/**
- * @param {MetricLabels} [labels]
- * @returns {MetricLabels}
- */
-function cloneLabels(labels = {}) {
-    return Object.freeze({ ...labels });
-}
-/**
- * @returns {Context.Tag.Service<MetricsService>}
- */
-export function makeInMemoryMetricsService() {
-    const registry = new Map();
-    const processStartMs = Date.now();
-    const asyncExternalWaitCounts = {
-        approval: 0,
-        event: 0,
-    };
-    /**
-   * @param {string} name
-   * @param {MetricLabels} [labels]
-   * @returns {CounterEntry}
-   */
-    function upsertCounter(name, labels) {
-        const key = metricKey(name, labels);
-        const existing = registry.get(key);
-        if (existing?.type === "counter")
-            return existing;
-        const created = {
-            type: "counter",
-            value: 0,
-            labels: cloneLabels(labels),
-        };
-        registry.set(key, created);
-        return created;
-    }
-    /**
-   * @param {string} name
-   * @param {MetricLabels} [labels]
-   * @returns {GaugeEntry}
-   */
-    function upsertGauge(name, labels) {
-        const key = metricKey(name, labels);
-        const existing = registry.get(key);
-        if (existing?.type === "gauge")
-            return existing;
-        const created = {
-            type: "gauge",
-            value: 0,
-            labels: cloneLabels(labels),
-        };
-        registry.set(key, created);
-        return created;
-    }
-    /**
-   * @param {string} name
-   * @param {MetricLabels} [labels]
-   * @returns {HistogramEntry}
-   */
-    function upsertHistogram(name, labels) {
-        const key = metricKey(name, labels);
-        const existing = registry.get(key);
-        if (existing?.type === "histogram")
-            return existing;
-        const created = {
-            type: "histogram",
-            sum: 0,
-            count: 0,
-            labels: cloneLabels(labels),
-            buckets: new Map(DEFAULT_HISTOGRAM_BUCKETS.map((bucket) => [bucket, 0])),
-        };
-        registry.set(key, created);
-        return created;
-    }
-    /**
-   * @returns {PrometheusSample[]}
-   */
-    function samples() {
-        return [...registry.entries()].map(([key, entry]) => {
-            const name = key.slice(0, key.indexOf("|"));
-            if (entry.type === "histogram") {
-                return {
-                    name,
-                    type: entry.type,
-                    labels: entry.labels,
-                    buckets: new Map(entry.buckets),
-                    sum: entry.sum,
-                    count: entry.count,
-                };
-            }
-            return {
-                name,
-                type: entry.type,
-                labels: entry.labels,
-                value: entry.value,
-            };
-        });
-    }
-    const service = {
-        increment: (name, labels) => service.incrementBy(name, 1, labels),
-        incrementBy: (name, value, labels) => Effect.sync(() => {
-            const key = metricKey(name, labels);
-            const existing = registry.get(key);
-            const definition = smithersMetricCatalogByName.get(name);
-            if (existing?.type === "gauge" || definition?.type === "gauge") {
-                upsertGauge(name, labels).value += value;
-                return;
-            }
-            upsertCounter(name, labels).value += value;
-        }),
-        gauge: (name, value, labels) => Effect.sync(() => {
-            upsertGauge(name, labels).value = value;
-        }),
-        histogram: (name, value, labels) => Effect.sync(() => {
-            const entry = upsertHistogram(name, labels);
-            entry.count += 1;
-            entry.sum += value;
-            for (const boundary of DEFAULT_HISTOGRAM_BUCKETS) {
-                if (value <= boundary) {
-                    entry.buckets.set(boundary, (entry.buckets.get(boundary) ?? 0) + 1);
-                }
-            }
-        }),
-        recordEvent: (event) => {
-            const eventType = String(event.type);
-            const countEvent = service.increment("smithers.events.emitted_total", {
-                type: eventType,
-            });
-            switch (event.type) {
-                case "RunStarted":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.runs.total"),
-                        service.incrementBy("smithers.runs.active", 1),
-                    ], { discard: true });
-                case "RunFinished":
-                    return Effect.all([
-                        countEvent,
-                        service.incrementBy("smithers.runs.active", -1),
-                        service.increment("smithers.runs.finished_total"),
-                    ], { discard: true });
-                case "RunFailed":
-                    return Effect.all([
-                        countEvent,
-                        service.incrementBy("smithers.runs.active", -1),
-                        service.increment("smithers.runs.failed_total"),
-                        service.increment("smithers.errors.total"),
-                    ], { discard: true });
-                case "RunCancelled":
-                    return Effect.all([
-                        countEvent,
-                        service.incrementBy("smithers.runs.active", -1),
-                        service.increment("smithers.runs.cancelled_total"),
-                    ], { discard: true });
-                case "RunContinuedAsNew":
-                    return Effect.all([countEvent, service.increment("smithers.runs.continued_total")], { discard: true });
-                case "NodeStarted":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.nodes.started"),
-                        service.incrementBy("smithers.nodes.active", 1),
-                    ], { discard: true });
-                case "NodeFinished":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.nodes.finished"),
-                        service.incrementBy("smithers.nodes.active", -1),
-                        typeof event.durationMs === "number"
-                            ? service.histogram("smithers.node.duration_ms", event.durationMs)
-                            : Effect.void,
-                    ], { discard: true });
-                case "NodeFailed":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.nodes.failed"),
-                        service.increment("smithers.errors.total"),
-                        service.incrementBy("smithers.nodes.active", -1),
-                    ], { discard: true });
-                case "CacheHit":
-                    return Effect.all([countEvent, service.increment("smithers.cache.hits")], { discard: true });
-                case "CacheMiss":
-                    return Effect.all([countEvent, service.increment("smithers.cache.misses")], { discard: true });
-                case "ApprovalRequested":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.approvals.requested"),
-                        service.incrementBy("smithers.approval.pending", 1),
-                    ], { discard: true });
-                case "ApprovalResolved": {
-                    const approved = event.approved === true || event.status === "approved";
-                    return Effect.all([
-                        countEvent,
-                        service.increment(approved
-                            ? "smithers.approvals.granted"
-                            : "smithers.approvals.denied"),
-                        service.incrementBy("smithers.approval.pending", -1),
-                    ], { discard: true });
-                }
-                case "TimerCreated":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.timers.created"),
-                        service.incrementBy("smithers.timers.pending", 1),
-                    ], { discard: true });
-                case "TimerFired":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.timers.fired"),
-                        service.incrementBy("smithers.timers.pending", -1),
-                    ], { discard: true });
-                case "TaskHeartbeat":
-                    return Effect.all([countEvent, service.increment("smithers.heartbeats.total")], { discard: true });
-                case "TaskHeartbeatTimeout":
-                    return Effect.all([
-                        countEvent,
-                        service.increment("smithers.heartbeats.timeout_total"),
-                        service.increment("smithers.errors.total"),
-                    ], { discard: true });
-                case "TokenUsageReported": {
-                    const effects = [countEvent];
-                    const labels = {
-                        ...(typeof event.agent === "string" ? { agent: event.agent } : {}),
-                        ...(typeof event.model === "string" ? { model: event.model } : {}),
-                    };
-                    /**
-           * @param {string} name
-           * @param {unknown} value
-           */
-                    const push = (name, value) => {
-                        if (typeof value === "number" && Number.isFinite(value) && value > 0) {
-                            effects.push(service.incrementBy(name, value, labels));
-                        }
-                    };
-                    push("smithers.tokens.input_total", event.inputTokens);
-                    push("smithers.tokens.output_total", event.outputTokens);
-                    push("smithers.tokens.cache_read_total", event.cacheReadTokens);
-                    push("smithers.tokens.cache_write_total", event.cacheWriteTokens);
-                    push("smithers.tokens.reasoning_total", event.reasoningTokens);
-                    return Effect.all(effects, { discard: true });
-                }
-                default:
-                    return countEvent;
-            }
-        },
-        updateProcessMetrics: () => Effect.sync(() => {
-            const uptimeS = (Date.now() - processStartMs) / 1000;
-            const mem = process.memoryUsage();
-            upsertGauge("smithers.process.uptime_seconds").value = uptimeS;
-            upsertGauge("smithers.process.memory_rss_bytes").value = mem.rss;
-            upsertGauge("smithers.process.heap_used_bytes").value = mem.heapUsed;
-        }),
-        updateAsyncExternalWaitPending: (kind, delta) => Effect.sync(() => {
-            asyncExternalWaitCounts[kind] = Math.max(0, asyncExternalWaitCounts[kind] + delta);
-            upsertGauge("smithers.external_wait.async_pending", { kind }).value =
-                asyncExternalWaitCounts[kind];
-        }),
-        renderPrometheus: () => Effect.sync(() => renderPrometheusSamples(samples())),
-        snapshot: () => Effect.sync(() => new Map(registry)),
-    };
-    return service;
-}
-/** @type {Layer.Layer<MetricsService, never, never>} */
-export const MetricsServiceLive = Layer.sync(MetricsService, makeInMemoryMetricsService);
-/** @type {Layer.Layer<MetricsService, never, never>} */
-export const MetricsServiceNoop = Layer.succeed(MetricsService, {
-    increment: () => Effect.void,
-    incrementBy: () => Effect.void,
-    gauge: () => Effect.void,
-    histogram: () => Effect.void,
-    recordEvent: () => Effect.void,
-    updateProcessMetrics: () => Effect.void,
-    updateAsyncExternalWaitPending: () => Effect.void,
-    renderPrometheus: () => Effect.succeed(""),
-    snapshot: () => Effect.succeed(new Map()),
-});

package/src/_coreTracing.js

@@ -1,5 +1,6 @@
 import { Context, Effect, Layer } from "effect";
 import { correlationContextToLogAnnotations, getCurrentCorrelationContext, withCorrelationContext, } from "./_coreCorrelation/index.js";
+import { smithersSpanAttributeAliases } from "./_smithersSpanAttributeAliases.js";
 /** @typedef {import("./SmithersLogFormat.ts").SmithersLogFormat} SmithersLogFormat */
 /** @typedef {import("./_coreTracingShape.ts").SmithersSpanAttributesInput} SmithersSpanAttributesInput */
 /** @typedef {import("./_coreTracingShape.ts").TracingServiceShape} TracingServiceShape */
@@ -30,25 +31,7 @@ export function getCurrentSmithersTraceAnnotations() {
  * @returns {Record<string, unknown>}
  */
 export function makeSmithersSpanAttributes(attributes = {}) {
-    const aliases = {
-        runId: "smithers.run_id",
-        run_id: "smithers.run_id",
-        workflowName: "smithers.workflow_name",
-        workflow_name: "smithers.workflow_name",
-        nodeId: "smithers.node_id",
-        node_id: "smithers.node_id",
-        iteration: "smithers.iteration",
-        attempt: "smithers.attempt",
-        nodeLabel: "smithers.node_label",
-        node_label: "smithers.node_label",
-        toolName: "smithers.tool_name",
-        tool_name: "smithers.tool_name",
-        agent: "smithers.agent",
-        model: "smithers.model",
-        status: "smithers.status",
-        waitReason: "smithers.wait_reason",
-        wait_reason: "smithers.wait_reason",
-    };
+    const aliases = smithersSpanAttributeAliases;
     const result = {};
     for (const [key, value] of Object.entries(attributes)) {
         if (value !== undefined) {
@@ -114,18 +97,22 @@ export function annotateSmithersTrace(attributes = {}) {
  */
 export function withSmithersSpan(name, effect, attributes) {
     const spanAttributes = makeSmithersSpanAttributes(attributes);
-    const annotations = correlationContextToLogAnnotations(getCurrentCorrelationContext());
-    let program = effect;
-    if (Object.keys(spanAttributes).length > 0) {
-        program = program.pipe(Effect.annotateSpans(spanAttributes));
-    }
-    if (hasAttributes(attributes)) {
-        program = program.pipe(Effect.annotateLogs(attributes));
-    }
-    if (annotations) {
-        program = program.pipe(Effect.annotateLogs(annotations));
-    }
-    return program.pipe(Effect.withLogSpan(name), Effect.withSpan(inferSmithersSpanName(name, attributes)));
+    return Effect.suspend(() => {
+        const annotations = correlationContextToLogAnnotations(getCurrentCorrelationContext());
+        let program = effect;
+        if (Object.keys(spanAttributes).length > 0) {
+            program = program.pipe(Effect.annotateSpans(spanAttributes));
+        }
+        if (hasAttributes(attributes)) {
+            program = program.pipe(Effect.annotateLogs(attributes));
+        }
+        if (annotations) {
+            program = program.pipe(Effect.annotateLogs(annotations));
+        }
+        return program.pipe(Effect.withLogSpan(name), Effect.withSpan(inferSmithersSpanName(name, attributes), {
+            attributes: spanAttributes,
+        }));
+    });
 }
 /** @type {Layer.Layer<TracingService, never, never>} */
 export const TracingServiceLive = Layer.succeed(TracingService, {

package/src/_traceEventNormalizers.js

@@ -1,6 +1,3 @@
-import { extractTextFromJsonValue } from "@smithers-orchestrator/agents/BaseCliAgent";
-import { normalizeTokenUsage } from "@smithers-orchestrator/agents/BaseCliAgent";
-
 /**
  * @typedef {import('./agentTrace.ts').AgentFamily} AgentFamily
  * @typedef {import('./agentTrace.ts').CanonicalAgentTraceEventKind} CanonicalAgentTraceEventKind
@@ -27,6 +24,77 @@ import { normalizeTokenUsage } from "@smithers-orchestrator/agents/BaseCliAgent"
  * }} NormalizedTraceBatch
  */
 
+/**
+ * @param {unknown} value
+ * @returns {string | undefined}
+ */
+function extractTextFromJsonValue(value) {
+    if (typeof value === "string") return value;
+    if (Array.isArray(value)) {
+        const text = value.map((item) => extractTextFromJsonValue(item) ?? "").join("");
+        return text || undefined;
+    }
+    if (!value || typeof value !== "object") return undefined;
+    const record = /** @type {Record<string, unknown>} */ (value);
+    if (typeof record.text === "string") return record.text;
+    if (typeof record.content === "string") return record.content;
+    if (typeof record.output_text === "string") return record.output_text;
+    if (Array.isArray(record.content)) {
+        const text = record.content.map((part) => extractTextFromJsonValue(part) ?? "").join("");
+        if (text) return text;
+    }
+    if (record.type === "text" && record.part) return extractTextFromJsonValue(record.part);
+    for (const field of ["response", "message", "result", "output", "data", "item"]) {
+        const text = extractTextFromJsonValue(record[field]);
+        if (text) return text;
+    }
+    return undefined;
+}
+
+/** @type {Record<string, ReadonlyArray<ReadonlyArray<string>>>} */
+const _usageFieldAliases = {
+    inputTokens: [["inputTokens"], ["promptTokens"], ["prompt_tokens"], ["input_tokens"], ["input"], ["models", "gemini", "tokens", "input"]],
+    outputTokens: [["outputTokens"], ["completionTokens"], ["completion_tokens"], ["output_tokens"], ["output"], ["models", "gemini", "tokens", "output"]],
+    cacheReadTokens: [["cacheReadTokens"], ["cache_read_input_tokens"], ["cached_input_tokens"], ["cache_read_tokens"], ["inputTokenDetails", "cacheReadTokens"]],
+    cacheWriteTokens: [["cacheWriteTokens"], ["cache_write_input_tokens"], ["cache_creation_input_tokens"], ["cache_write_tokens"], ["inputTokenDetails", "cacheWriteTokens"]],
+    reasoningTokens: [["reasoningTokens"], ["reasoning_tokens"], ["outputTokenDetails", "reasoningTokens"]],
+    totalTokens: [["totalTokens"], ["total_tokens"]],
+};
+
+/**
+ * @param {unknown} value
+ * @param {ReadonlyArray<string>} path
+ * @returns {unknown}
+ */
+function _readUsagePath(value, path) {
+    let current = value;
+    for (const segment of path) {
+        if (!current || typeof current !== "object") return undefined;
+        current = /** @type {Record<string, unknown>} */ (current)[segment];
+    }
+    return current;
+}
+
+/**
+ * @param {unknown} usage
+ * @returns {Record<string, number> | null}
+ */
+function normalizeTokenUsage(usage) {
+    if (!usage || typeof usage !== "object") return null;
+    /** @type {Record<string, number>} */
+    const normalized = {};
+    for (const [field, aliases] of Object.entries(_usageFieldAliases)) {
+        for (const path of aliases) {
+            const value = _readUsagePath(usage, path);
+            if (typeof value === "number") {
+                normalized[field] = value;
+                break;
+            }
+        }
+    }
+    return Object.values(normalized).some((v) => Number.isFinite(v) && v > 0) ? normalized : null;
+}
+
 /** @type {Record<string, MappedStructuredEvent>} */
 const piSimpleEventMap = {
     session: { kind: "session.start", payloadKind: "pi" },

package/src/_traceRedaction.js

@@ -27,7 +27,8 @@ const rules = [
     {
         id: "secret-ish",
         pattern: /\b(?:api[_-]?key|token|secret|password)=([^\s"']+)/gi,
-        replace: "",
+        // No `replace` field: redactValue special-cases this rule by id and
+        // rewrites the captured value itself, so a top-level replace is never read.
     },
 ];
 

package/src/createSmithersObservabilityLayer.js

@@ -45,5 +45,11 @@ function makeService(options) {
  */
 export function createSmithersObservabilityLayer(options = {}) {
     const resolved = resolveSmithersObservabilityOptions(options);
-    return Layer.mergeAll(BunContext.layer, Logger.replace(Logger.defaultLogger, resolveLogger(resolved.logFormat)), Logger.minimumLogLevel(resolved.logLevel), createSmithersOtelLayer(resolved), MetricsServiceLive, TracingServiceLive, Layer.succeed(SmithersObservability, makeService(resolved)));
+    const loggerLayers = resolved.installLogger
+        ? [
+            Logger.replace(Logger.defaultLogger, resolveLogger(resolved.logFormat)),
+            Logger.minimumLogLevel(resolved.logLevel),
+        ]
+        : [];
+    return Layer.mergeAll(BunContext.layer, ...loggerLayers, createSmithersOtelLayer(resolved), MetricsServiceLive, TracingServiceLive, Layer.succeed(SmithersObservability, makeService(resolved)));
 }

package/src/index.d.ts

@@ -13,6 +13,7 @@ type ResolvedSmithersObservabilityOptions$2 = {
     readonly serviceName: string;
     readonly logFormat: SmithersLogFormat$1;
     readonly logLevel: LogLevel.LogLevel;
+    readonly installLogger: boolean;
 };
 
 type SmithersObservabilityService$1 = {
@@ -27,6 +28,7 @@ type SmithersObservabilityOptions$4 = {
     readonly serviceName?: string;
     readonly logFormat?: SmithersLogFormat$1;
     readonly logLevel?: LogLevel.LogLevel | string;
+    readonly installLogger?: boolean;
 };
 
 type MetricLabels$1 = Readonly<Record<string, string | number | boolean>>;
@@ -47,6 +49,89 @@ type SmithersMetricDefinition$2 = {
     readonly boundaries?: readonly number[];
 };
 
+type AgentFamily = "pi" | "codex" | "claude-code" | "antigravity" | "gemini" | "kimi" | "openai" | "anthropic" | "amp" | "forge" | "unknown";
+type AgentCaptureMode = "sdk-events" | "rpc-events" | "cli-json-stream" | "cli-json" | "cli-text" | "artifact-import";
+type TraceCompleteness = "full-observed" | "partial-observed" | "final-only" | "capture-failed";
+type CanonicalAgentTraceEventKind = "session.start" | "session.end" | "turn.start" | "turn.end" | "message.start" | "message.update" | "message.end" | "assistant.text.delta" | "assistant.thinking.delta" | "assistant.message.final" | "tool.execution.start" | "tool.execution.update" | "tool.execution.end" | "tool.result" | "retry.start" | "retry.end" | "compaction.start" | "compaction.end" | "stderr" | "stdout" | "usage" | "capture.warning" | "capture.error" | "artifact.created";
+type CanonicalAgentTraceEventPhase = "agent" | "turn" | "message" | "tool" | "session" | "capture" | "artifact";
+type CanonicalAgentTraceEvent = {
+    traceVersion: "1";
+    runId: string;
+    workflowPath?: string;
+    workflowHash?: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    timestampMs: number;
+    event: {
+        sequence: number;
+        kind: CanonicalAgentTraceEventKind;
+        phase: CanonicalAgentTraceEventPhase;
+    };
+    source: {
+        agentFamily: AgentFamily;
+        captureMode: AgentCaptureMode;
+        rawType?: string;
+        rawEventId?: string;
+        observed: boolean;
+    };
+    traceCompleteness: TraceCompleteness;
+    payload: Record<string, unknown> | null;
+    raw: unknown;
+    redaction: {
+        applied: boolean;
+        ruleIds: string[];
+    };
+    annotations: Record<string, string | number | boolean>;
+};
+type AgentTraceSummary = {
+    traceVersion: "1";
+    runId: string;
+    workflowPath?: string;
+    workflowHash?: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    traceStartedAtMs: number;
+    traceFinishedAtMs: number;
+    agentFamily: AgentFamily;
+    agentId?: string;
+    model?: string;
+    captureMode: AgentCaptureMode;
+    traceCompleteness: TraceCompleteness;
+    unsupportedEventKinds: CanonicalAgentTraceEventKind[];
+    missingExpectedEventKinds: CanonicalAgentTraceEventKind[];
+    rawArtifactRefs: string[];
+};
+type AgentSessionTranscriptEvent = {
+    transcriptVersion: "1";
+    runId: string;
+    workflowPath?: string;
+    workflowHash?: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    timestampMs: number;
+    event: {
+        sequence: number;
+        rowType: string;
+    };
+    source: {
+        agentFamily: AgentFamily;
+        captureMode: AgentCaptureMode;
+        ingestSource: "live" | "artifact";
+        observedLive: boolean;
+        providerSessionId?: string;
+        providerThreadId?: string;
+    };
+    raw: unknown;
+    redaction: {
+        applied: boolean;
+        ruleIds: string[];
+    };
+    annotations: Record<string, string | number | boolean>;
+};
+
 type RunStatus = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "finished" | "continued" | "failed" | "cancelled";
 type RunState = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "recovering" | "stale" | "orphaned" | "failed" | "cancelled" | "succeeded" | "unknown";
 type AgentCliActionKind = "turn" | "command" | "tool" | "file_change" | "web_search" | "todo_list" | "reasoning" | "warning" | "note";
@@ -128,6 +213,14 @@ type SmithersEvent$2 = {
     type: "RunFinished";
     runId: string;
     timestampMs: number;
+    /**
+     * Tasks that ended `failed` but were tolerated (continueOnFail / transient
+     * agent failures) so the run still finished. Present only when `> 0` — the
+     * run "succeeded" while these children failed. See `docs/runtime/run-state.mdx`.
+     */
+    failedChildren?: number;
+    /** Task state keys (`nodeId::iteration`) counted by `failedChildren`. */
+    failedChildKeys?: readonly string[];
 } | {
     type: "RunFailed";
     runId: string;
@@ -230,6 +323,11 @@ type SmithersEvent$2 = {
     runId: string;
     frameNo: number;
     xmlHash: string;
+    trigger?: {
+        reason: string;
+        nodeId?: string;
+        iteration?: number;
+    };
     timestampMs: number;
 } | {
     type: "NodePending";
@@ -565,6 +663,30 @@ type SmithersEvent$2 = {
     runId: string;
     timerId: string;
     timestampMs: number;
+} | {
+    type: "AgentTraceEvent";
+    runId: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    trace: CanonicalAgentTraceEvent;
+    timestampMs: number;
+} | {
+    type: "AgentTraceSummary";
+    runId: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    summary: AgentTraceSummary;
+    timestampMs: number;
+} | {
+    type: "AgentSessionEvent";
+    runId: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    transcript: AgentSessionTranscriptEvent;
+    timestampMs: number;
 };
 
 type MetricName = string;
@@ -995,11 +1117,40 @@ declare function correlationContextToLogAnnotations(context?: CorrelationContext
 type CorrelationContext$1 = CorrelationContext$5;
 
 /**
+ * Temporary compatibility shim for legacy, non-Effect callers.
+ *
+ * Unlike the FiberRef-based core implementation
+ * ({@link import("./_coreCorrelation/updateCurrentCorrelationContext.js").updateCurrentCorrelationContext}),
+ * which returns an Effect and sets a fresh merged context on the
+ * `correlationContextFiberRef`, this shim runs synchronously and applies the
+ * patch by **mutating the current context object in place** via
+ * `Object.assign(current, next)`. Any references already holding the current
+ * context object will observe the mutation. This in-place semantics is
+ * intentional and exists only to preserve behavior for callers that captured a
+ * context reference before the Effect-based API existed.
+ *
+ * If there is no current context, the patch is a no-op (nothing is created).
+ *
+ * @deprecated Prefer the Effect-returning
+ * `updateCurrentCorrelationContext` from
+ * `@smithers-orchestrator/observability` (the `_coreCorrelation` version),
+ * which does not mutate shared state. This shim will be removed once legacy
+ * callers migrate.
+ *
  * @param {CorrelationPatch} patch
+ * @returns {void}
  */
 declare function updateCurrentCorrelationContext(patch: CorrelationPatch$1): void;
 type CorrelationPatch$1 = CorrelationPatch$5;
 
+/**
+ * Install the Effect runtime used by fire-and-forget observability logs.
+ * Returns a restore function so tests and embedded hosts can scope overrides.
+ *
+ * @param {SmithersLogRunner | null} runner
+ * @returns {() => void}
+ */
+declare function setSmithersLogRunner(runner: SmithersLogRunner | null): () => void;
 /**
  * @param {string} message
  * @param {LogAnnotations} [annotations]
@@ -1024,6 +1175,10 @@ declare function logWarning(message: string, annotations?: LogAnnotations, span?
  * @param {string} [span]
  */
 declare function logError(message: string, annotations?: LogAnnotations, span?: string): void;
+type SmithersLogRunner = {
+    runFork: (effect: Effect.Effect<void, never, never>) => unknown;
+    runPromise: (effect: Effect.Effect<void, never, never>) => Promise<void>;
+};
 type LogAnnotations = Record<string, unknown> | undefined;
 
 type CorrelationContext = CorrelationContext$5;
@@ -1039,4 +1194,4 @@ type SmithersMetricDefinition = SmithersMetricDefinition$2;
 type SmithersObservabilityOptions = SmithersObservabilityOptions$4;
 type SmithersObservabilityService = SmithersObservabilityService$1;
 
-export { type CorrelationContext, CorrelationContextLive, type CorrelationContextPatch, CorrelationContextService, type CorrelationPatch, type MetricLabels, MetricsService, MetricsServiceLive, type MetricsServiceShape, type MetricsSnapshot, type ResolvedSmithersObservabilityOptions, type SmithersEvent, type SmithersLogFormat, type SmithersMetricDefinition, SmithersObservability, type SmithersObservabilityOptions, type SmithersObservabilityService, TracingService, TracingServiceLive, activeNodes, activeRuns, annotateSmithersTrace, approvalPending, approvalWaitDuration, approvalsDenied, approvalsGranted, approvalsRequested, attemptDuration, cacheHits, cacheMisses, correlationContextFiberRef, correlationContextToLogAnnotations, createSmithersObservabilityLayer, createSmithersOtelLayer, createSmithersRuntimeLayer, dbQueryDuration, dbRetries, dbTransactionDuration, dbTransactionRetries, dbTransactionRollbacks, errorsTotal, eventsEmittedTotal, externalWaitAsyncPending, getCurrentCorrelationContext, getCurrentCorrelationContextEffect, getCurrentSmithersTraceAnnotations, getCurrentSmithersTraceSpan, hotReloadDuration, hotReloadFailures, hotReloads, httpRequestDuration, httpRequests, logDebug, logError, logInfo, logWarning, makeSmithersSpanAttributes, mergeCorrelationContext, metricsServiceAdapter, nodeDuration, nodeRetriesTotal, nodesFailed, nodesFinished, nodesStarted, processHeapUsedBytes, processMemoryRssBytes, processUptimeSeconds, prometheusContentType, promptSizeBytes, renderPrometheusMetrics, replaysStarted, resolveSmithersObservabilityOptions, responseSizeBytes, rewindDurationMs, rewindFramesDeleted, rewindRollbackTotal, rewindSandboxesReverted, rewindTotal, runDuration, runForksCreated, runWithCorrelationContext, runsAncestryDepth, runsCancelledTotal, runsCarriedStateBytes, runsContinuedTotal, runsFailedTotal, runsFinishedTotal, runsResumedTotal, runsTotal, sandboxActive, sandboxBundleSizeBytes, sandboxCompletedTotal, sandboxCreatedTotal, sandboxDurationMs, sandboxPatchCount, sandboxTransportDurationMs, schedulerConcurrencyUtilization, schedulerQueueDepth, schedulerWaitDuration, scorerEventsFailed, scorerEventsFinished, scorerEventsStarted, smithersMetricCatalog, smithersMetrics, smithersSpanNames, snapshotDuration, snapshotsCaptured, timerDelayDuration, timersCancelled, timersCreated, timersFired, timersPending, toPrometheusMetricName, tokensCacheReadTotal, tokensCacheWriteTotal, tokensContextWindowBucketTotal, tokensContextWindowPerCall, tokensInputPerCall, tokensInputTotal, tokensOutputPerCall, tokensOutputTotal, tokensReasoningTotal, toolCallErrorsTotal, toolCallsTotal, toolDuration, toolOutputTruncatedTotal, trackEvent as trackSmithersEvent, updateCurrentCorrelationContext, updateProcessMetrics, vcsDuration, withCorrelationContext, withCurrentCorrelationContext, withSmithersSpan };
+export { type CorrelationContext, CorrelationContextLive, type CorrelationContextPatch, CorrelationContextService, type CorrelationPatch, type MetricLabels, MetricsService, MetricsServiceLive, type MetricsServiceShape, type MetricsSnapshot, type ResolvedSmithersObservabilityOptions, type SmithersEvent, type SmithersLogFormat, type SmithersMetricDefinition, SmithersObservability, type SmithersObservabilityOptions, type SmithersObservabilityService, TracingService, TracingServiceLive, activeNodes, activeRuns, annotateSmithersTrace, approvalPending, approvalWaitDuration, approvalsDenied, approvalsGranted, approvalsRequested, attemptDuration, cacheHits, cacheMisses, correlationContextFiberRef, correlationContextToLogAnnotations, createSmithersObservabilityLayer, createSmithersOtelLayer, createSmithersRuntimeLayer, dbQueryDuration, dbRetries, dbTransactionDuration, dbTransactionRetries, dbTransactionRollbacks, errorsTotal, eventsEmittedTotal, externalWaitAsyncPending, getCurrentCorrelationContext, getCurrentCorrelationContextEffect, getCurrentSmithersTraceAnnotations, getCurrentSmithersTraceSpan, hotReloadDuration, hotReloadFailures, hotReloads, httpRequestDuration, httpRequests, logDebug, logError, logInfo, logWarning, makeSmithersSpanAttributes, mergeCorrelationContext, metricsServiceAdapter, nodeDuration, nodeRetriesTotal, nodesFailed, nodesFinished, nodesStarted, processHeapUsedBytes, processMemoryRssBytes, processUptimeSeconds, prometheusContentType, promptSizeBytes, renderPrometheusMetrics, replaysStarted, resolveSmithersObservabilityOptions, responseSizeBytes, rewindDurationMs, rewindFramesDeleted, rewindRollbackTotal, rewindSandboxesReverted, rewindTotal, runDuration, runForksCreated, runWithCorrelationContext, runsAncestryDepth, runsCancelledTotal, runsCarriedStateBytes, runsContinuedTotal, runsFailedTotal, runsFinishedTotal, runsResumedTotal, runsTotal, sandboxActive, sandboxBundleSizeBytes, sandboxCompletedTotal, sandboxCreatedTotal, sandboxDurationMs, sandboxPatchCount, sandboxTransportDurationMs, schedulerConcurrencyUtilization, schedulerQueueDepth, schedulerWaitDuration, scorerEventsFailed, scorerEventsFinished, scorerEventsStarted, setSmithersLogRunner, smithersMetricCatalog, smithersMetrics, smithersSpanNames, snapshotDuration, snapshotsCaptured, timerDelayDuration, timersCancelled, timersCreated, timersFired, timersPending, toPrometheusMetricName, tokensCacheReadTotal, tokensCacheWriteTotal, tokensContextWindowBucketTotal, tokensContextWindowPerCall, tokensInputPerCall, tokensInputTotal, tokensOutputPerCall, tokensOutputTotal, tokensReasoningTotal, toolCallErrorsTotal, toolCallsTotal, toolDuration, toolOutputTruncatedTotal, trackEvent as trackSmithersEvent, updateCurrentCorrelationContext, updateProcessMetrics, vcsDuration, withCorrelationContext, withCurrentCorrelationContext, withSmithersSpan };

package/src/index.js

@@ -32,4 +32,4 @@ export { rewindTotal, rewindRollbackTotal, rewindDurationMs, rewindFramesDeleted
 export { activeNodes, activeRuns, approvalPending, externalWaitAsyncPending, approvalsDenied, approvalsGranted, approvalsRequested, approvalWaitDuration, timerDelayDuration, timersCancelled, timersCreated, timersFired, timersPending, attemptDuration, cacheHits, cacheMisses, dbQueryDuration, dbRetries, dbTransactionDuration, dbTransactionRetries, dbTransactionRollbacks, errorsTotal, eventsEmittedTotal, hotReloadDuration, hotReloadFailures, hotReloads, httpRequestDuration, httpRequests, nodeDuration, nodeRetriesTotal, nodesFailed, nodesFinished, nodesStarted, processHeapUsedBytes, processMemoryRssBytes, processUptimeSeconds, promptSizeBytes, responseSizeBytes, runDuration, runsCancelledTotal, runsContinuedTotal, runsFailedTotal, runsFinishedTotal, runsResumedTotal, runsAncestryDepth, runsCarriedStateBytes, sandboxActive, sandboxBundleSizeBytes, sandboxCompletedTotal, sandboxCreatedTotal, sandboxDurationMs, sandboxPatchCount, sandboxTransportDurationMs, runsTotal, schedulerConcurrencyUtilization, schedulerQueueDepth, schedulerWaitDuration, tokensCacheReadTotal, tokensCacheWriteTotal, tokensContextWindowBucketTotal, tokensContextWindowPerCall, tokensInputPerCall, tokensInputTotal, tokensOutputPerCall, tokensOutputTotal, tokensReasoningTotal, toolCallErrorsTotal, toolCallsTotal, toolDuration, toolOutputTruncatedTotal, scorerEventsStarted, scorerEventsFinished, scorerEventsFailed, snapshotsCaptured, runForksCreated, replaysStarted, snapshotDuration, trackEvent as trackSmithersEvent, updateProcessMetrics, vcsDuration, toPrometheusMetricName, smithersMetricCatalog, metricsServiceAdapter, } from "./metrics/index.js";
 export { correlationContextFiberRef, correlationContextToLogAnnotations, CorrelationContextLive, CorrelationContextService, getCurrentCorrelationContext, getCurrentCorrelationContextEffect, mergeCorrelationContext, runWithCorrelationContext, withCorrelationContext, withCurrentCorrelationContext, } from "./correlation.js";
 export { updateCurrentCorrelationContext } from "./correlation.js";
-export { logDebug, logInfo, logWarning, logError, } from "./logging.js";
+export { logDebug, logInfo, logWarning, logError, setSmithersLogRunner, } from "./logging.js";

package/src/logging.js

@@ -1,4 +1,4 @@
-import { Effect } from "effect";
+import { Effect, Logger, LogLevel } from "effect";
 import { getCurrentSmithersTraceAnnotations } from "./getCurrentSmithersTraceAnnotations.js";
 import { correlationContextToLogAnnotations, getCurrentCorrelationContext, withCurrentCorrelationContext, } from "./correlation.js";
 /**
@@ -11,6 +11,7 @@ const LOG_LEVEL_DEBUG = 1;
 const LOG_LEVEL_INFO = 2;
 const LOG_LEVEL_WARNING = 3;
 const LOG_LEVEL_ERROR = 4;
+const LOG_RUNNER_KEY = Symbol.for("smithers.observability.logRunner");
 
 /** @returns {number} */
 function resolveMinLevel() {
@@ -31,6 +32,58 @@ function resolveMinLevel() {
 
 const minLevel = resolveMinLevel();
 
+/**
+ * @typedef {{
+ *   runFork: (effect: Effect.Effect<void, never, never>) => unknown;
+ *   runPromise: (effect: Effect.Effect<void, never, never>) => Promise<void>;
+ * }} SmithersLogRunner
+ */
+
+/** @returns {{ runner: SmithersLogRunner | null }} */
+function getRunnerState() {
+    const globalState = /** @type {typeof globalThis & { [LOG_RUNNER_KEY]?: { runner: SmithersLogRunner | null } }} */ (globalThis);
+    globalState[LOG_RUNNER_KEY] ??= { runner: null };
+    return globalState[LOG_RUNNER_KEY];
+}
+
+/** @type {SmithersLogRunner} */
+const defaultRunner = {
+    runFork: (effect) => Effect.runFork(effect),
+    runPromise: (effect) => Effect.runPromise(effect),
+};
+
+/** @returns {SmithersLogRunner} */
+function getLogRunner() {
+    return getRunnerState().runner ?? defaultRunner;
+}
+
+/**
+ * Install the Effect runtime used by fire-and-forget observability logs.
+ * Returns a restore function so tests and embedded hosts can scope overrides.
+ *
+ * @param {SmithersLogRunner | null} runner
+ * @returns {() => void}
+ */
+export function setSmithersLogRunner(runner) {
+    const state = getRunnerState();
+    const previous = state.runner;
+    state.runner = runner;
+    return () => {
+        state.runner = previous;
+    };
+}
+
+/** @param {number} level */
+function toEffectLogLevel(level) {
+    switch (level) {
+        case LOG_LEVEL_DEBUG: return LogLevel.Debug;
+        case LOG_LEVEL_INFO: return LogLevel.Info;
+        case LOG_LEVEL_WARNING: return LogLevel.Warning;
+        case LOG_LEVEL_ERROR: return LogLevel.Error;
+        default: return LogLevel.All;
+    }
+}
+
 /**
  * @param {Effect.Effect<void, never, never>} effect
  * @param {LogAnnotations} [annotations]
@@ -66,7 +119,12 @@ function buildLogProgram(effect, annotations, span) {
 function emitLog(effect, annotations, span, level = LOG_LEVEL_INFO) {
     if (level < minLevel) return;
     const program = buildLogProgram(effect, annotations, span);
-    if (program) void Effect.runFork(program);
+    if (!program) return;
+    try {
+        void getLogRunner().runFork(program.pipe(Logger.withMinimumLogLevel(toEffectLogLevel(level))));
+    } catch {
+        // Logging must never break the caller.
+    }
 }
 
 /**
@@ -79,7 +137,12 @@ function emitLog(effect, annotations, span, level = LOG_LEVEL_INFO) {
 async function emitLogAwait(effect, annotations, span, level = LOG_LEVEL_INFO) {
     if (level < minLevel) return;
     const program = buildLogProgram(effect, annotations, span);
-    if (program) await Effect.runPromise(program);
+    if (!program) return;
+    try {
+        await getLogRunner().runPromise(program.pipe(Logger.withMinimumLogLevel(toEffectLogLevel(level))));
+    } catch {
+        // Logging must never break the caller.
+    }
 }
 /**
  * @param {string} message

package/src/metrics/gatewayRunEventBackpressureDisconnectTotal.js

@@ -0,0 +1,2 @@
+import { Metric } from "effect";
+export const gatewayRunEventBackpressureDisconnectTotal = Metric.counter("smithers.gateway.run_event_backpressure_disconnect_total");

package/src/metrics/index.js

@@ -92,6 +92,7 @@ export { gatewayWebhooksRejectedTotal } from "./gatewayWebhooksRejectedTotal.js"
 export { devtoolsSubscribeTotal } from "./devtoolsSubscribeTotal.js";
 export { devtoolsEventTotal } from "./devtoolsEventTotal.js";
 export { devtoolsBackpressureDisconnectTotal } from "./devtoolsBackpressureDisconnectTotal.js";
+export { gatewayRunEventBackpressureDisconnectTotal } from "./gatewayRunEventBackpressureDisconnectTotal.js";
 export { eventsEmittedTotal } from "./eventsEmittedTotal.js";
 export { taskHeartbeatsTotal } from "./taskHeartbeatsTotal.js";
 export { taskHeartbeatTimeoutTotal } from "./taskHeartbeatTimeoutTotal.js";

package/src/resolveSmithersObservabilityOptions.js

@@ -75,5 +75,6 @@ export function resolveSmithersObservabilityOptions(options = {}) {
             ? resolveLogFormat(options.logFormat)
             : resolveLogFormat(process.env.SMITHERS_LOG_FORMAT),
         logLevel: resolveLogLevel(options.logLevel ?? process.env.SMITHERS_LOG_LEVEL),
+        installLogger: options.installLogger !== false,
     };
 }