@smthrs/observability 0.34.0 → 0.35.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smthrs/observability",
-  "version": "0.34.0",
+  "version": "0.35.0",
   "description": "Concrete Smithers metrics, logging, tracing, and observability integrations",
   "type": "module",
   "sideEffects": false,

package/src/SmithersEvent.ts

@@ -25,6 +25,7 @@ type RunState =
   | "failed"
   | "cancelled"
   | "succeeded"
+  | "succeeded-with-failures"
   | "unknown";
 
 type AgentCliActionKind =
@@ -77,6 +78,15 @@ type AgentCliCompletedEvent = {
 
 type AgentCliEvent = AgentCliStartedEvent | AgentCliActionEvent | AgentCliCompletedEvent;
 
+export type RunCancellationSource = {
+  kind: "signal" | "rpc" | "cli" | "engine";
+  detail?: string;
+  signal?: string;
+  clientPid?: number;
+  requestId?: string;
+  clientIdentity?: string;
+};
+
 export type SmithersEvent =
   | {
       type: "SupervisorStarted";
@@ -121,6 +131,20 @@ export type SmithersEvent =
       after: RunState;
       timestampMs: number;
     }
+  | {
+      type: "RunConcurrencySaturated";
+      runId: string;
+      requestedDemand: number;
+      effectiveCap: number;
+      remediationCommand: string;
+      /** Present when a lifecycle-linked descendant caused the owner's budget to saturate. */
+      descendantRunId?: string;
+      /** Which owner budget bound admission. Historical events omit this. */
+      budget?: "run" | "subtree";
+      /** Present when `budget` is `subtree`. */
+      subtreeGroupId?: string;
+      timestampMs: number;
+    }
   | {
       type: "RunFinished";
       runId: string;
@@ -135,7 +159,13 @@ export type SmithersEvent =
       failedChildKeys?: readonly string[];
     }
   | { type: "RunFailed"; runId: string; error: unknown; timestampMs: number }
-  | { type: "RunCancelled"; runId: string; timestampMs: number }
+  | {
+      type: "RunCancelled";
+      runId: string;
+      timestampMs: number;
+      /** Missing only on historical or unattributed cancellation events. */
+      source?: RunCancellationSource;
+    }
   | {
       type: "RunContinuedAsNew";
       runId: string;
@@ -365,6 +395,17 @@ export type SmithersEvent =
       error: unknown;
       timestampMs: number;
     }
+  | {
+      type: "NodeStalled";
+      runId: string;
+      nodeId: string;
+      iteration: number;
+      attempt: number;
+      identicalFailures: number;
+      signature: string;
+      error: unknown;
+      timestampMs: number;
+    }
   | {
       type: "NodeCancelled";
       runId: string;
@@ -685,10 +726,14 @@ export type SmithersEvent =
       model: string;
       agent: string;
       inputTokens: number;
+      /** Non-cached input tokens. Falls back to inputTokens when the provider omits the breakdown. */
+      freshInputTokens?: number;
       outputTokens: number;
       cacheReadTokens?: number;
       cacheWriteTokens?: number;
       reasoningTokens?: number;
+      /** Estimated USD cost from Smithers' built-in model price table. */
+      costUsd?: number;
       timestampMs: number;
     }
   | {

package/src/index.d.ts

@@ -133,7 +133,7 @@ type AgentSessionTranscriptEvent = {
 };
 
 type RunStatus = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "waiting-quota" | "paused" | "finished" | "continued" | "failed" | "cancelled";
-type RunState = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "waiting-quota" | "paused" | "recovering" | "stale" | "orphaned" | "failed" | "cancelled" | "succeeded" | "unknown";
+type RunState = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "waiting-quota" | "paused" | "recovering" | "stale" | "orphaned" | "failed" | "cancelled" | "succeeded" | "succeeded-with-failures" | "unknown";
 type AgentCliActionKind = "turn" | "command" | "tool" | "file_change" | "web_search" | "todo_list" | "reasoning" | "warning" | "note";
 type AgentCliActionPhase = "started" | "updated" | "completed";
 type AgentCliEventLevel = "debug" | "info" | "warning" | "error";
@@ -169,6 +169,14 @@ type AgentCliCompletedEvent = {
     usage?: Record<string, unknown>;
 };
 type AgentCliEvent = AgentCliStartedEvent | AgentCliActionEvent | AgentCliCompletedEvent;
+type RunCancellationSource$1 = {
+    kind: "signal" | "rpc" | "cli" | "engine";
+    detail?: string;
+    signal?: string;
+    clientPid?: number;
+    requestId?: string;
+    clientIdentity?: string;
+};
 type SmithersEvent$2 = {
     type: "SupervisorStarted";
     runId: string;
@@ -209,6 +217,19 @@ type SmithersEvent$2 = {
     before: RunState;
     after: RunState;
     timestampMs: number;
+} | {
+    type: "RunConcurrencySaturated";
+    runId: string;
+    requestedDemand: number;
+    effectiveCap: number;
+    remediationCommand: string;
+    /** Present when a lifecycle-linked descendant caused the owner's budget to saturate. */
+    descendantRunId?: string;
+    /** Which owner budget bound admission. Historical events omit this. */
+    budget?: "run" | "subtree";
+    /** Present when `budget` is `subtree`. */
+    subtreeGroupId?: string;
+    timestampMs: number;
 } | {
     type: "RunFinished";
     runId: string;
@@ -230,6 +251,8 @@ type SmithersEvent$2 = {
     type: "RunCancelled";
     runId: string;
     timestampMs: number;
+    /** Missing only on historical or unattributed cancellation events. */
+    source?: RunCancellationSource$1;
 } | {
     type: "RunContinuedAsNew";
     runId: string;
@@ -433,6 +456,16 @@ type SmithersEvent$2 = {
     attempt: number;
     error: unknown;
     timestampMs: number;
+} | {
+    type: "NodeStalled";
+    runId: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    identicalFailures: number;
+    signature: string;
+    error: unknown;
+    timestampMs: number;
 } | {
     type: "NodeCancelled";
     runId: string;
@@ -719,10 +752,14 @@ type SmithersEvent$2 = {
     model: string;
     agent: string;
     inputTokens: number;
+    /** Non-cached input tokens. Falls back to inputTokens when the provider omits the breakdown. */
+    freshInputTokens?: number;
     outputTokens: number;
     cacheReadTokens?: number;
     cacheWriteTokens?: number;
     reasoningTokens?: number;
+    /** Estimated USD cost from Smithers' built-in model price table. */
+    costUsd?: number;
     timestampMs: number;
 } | {
     type: "SnapshotCaptured";
@@ -1289,6 +1326,17 @@ type CorrelationContext$1 = CorrelationContext$5;
 declare function updateCurrentCorrelationContext(patch: CorrelationPatch$1): void;
 type CorrelationPatch$1 = CorrelationPatch$5;
 
+/**
+ * Scope imperative Smithers logs to one host-owned Effect runner. Unlike
+ * {@link setSmithersLogRunner}, this is concurrency-safe: parallel embedded
+ * engine instances keep their loggers isolated across awaits and timers.
+ *
+ * @template T
+ * @param {SmithersLogRunner} runner
+ * @param {() => T} execute
+ * @returns {T}
+ */
+declare function runWithSmithersLogRunner<T>(runner: SmithersLogRunner, execute: () => T): T;
 /**
  * Install the Effect runtime used by fire-and-forget observability logs.
  * Returns a restore function so tests and embedded hosts can scope overrides.
@@ -1334,10 +1382,11 @@ type MetricLabels = MetricLabels$1;
 type MetricsServiceShape = MetricsServiceShape$3;
 type MetricsSnapshot = MetricsSnapshot$1;
 type ResolvedSmithersObservabilityOptions = ResolvedSmithersObservabilityOptions$2;
+type RunCancellationSource = RunCancellationSource$1;
 type SmithersEvent = SmithersEvent$2;
 type SmithersLogFormat = SmithersLogFormat$1;
 type SmithersMetricDefinition = SmithersMetricDefinition$2;
 type SmithersObservabilityOptions = SmithersObservabilityOptions$4;
 type SmithersObservabilityService = SmithersObservabilityService$2;
 
-export { type CorrelationContext, CorrelationContextLive, type CorrelationContextPatch, CorrelationContextService, type CorrelationPatch, type MetricLabels, MetricsService, MetricsServiceLive, type MetricsServiceShape, type MetricsSnapshot, type ResolvedSmithersObservabilityOptions, type SmithersEvent, type SmithersLogFormat, type SmithersMetricDefinition, SmithersObservability, type SmithersObservabilityOptions, type SmithersObservabilityService, TracingService, TracingServiceLive, activeNodes, activeRuns, annotateSmithersTrace, approvalPending, approvalWaitDuration, approvalsDenied, approvalsGranted, approvalsRequested, attemptDuration, cacheHits, cacheMisses, correlationContextFiberRef, correlationContextToLogAnnotations, createSmithersObservabilityLayer, createSmithersOtelLayer, createSmithersRuntimeLayer, dbQueryDuration, dbRetries, dbTransactionDuration, dbTransactionRetries, dbTransactionRollbacks, errorsTotal, eventsEmittedTotal, externalWaitAsyncPending, getCurrentCorrelationContext, getCurrentCorrelationContextEffect, getCurrentSmithersTraceAnnotations, getCurrentSmithersTraceSpan, hotReloadDuration, hotReloadFailures, hotReloads, httpRequestDuration, httpRequests, logDebug, logError, logInfo, logWarning, makeSmithersSpanAttributes, mergeCorrelationContext, metricsServiceAdapter, nodeDuration, nodeRetriesTotal, nodesFailed, nodesFinished, nodesStarted, processHeapUsedBytes, processMemoryRssBytes, processUptimeSeconds, prometheusContentType, promptSizeBytes, renderPrometheusMetrics, replaysStarted, resolveSmithersObservabilityOptions, responseSizeBytes, rewindDurationMs, rewindFramesDeleted, rewindRollbackTotal, rewindSandboxesReverted, rewindTotal, runDuration, runForksCreated, runWithCorrelationContext, runsAncestryDepth, runsCancelledTotal, runsCarriedStateBytes, runsContinuedTotal, runsFailedTotal, runsFinishedTotal, runsResumedTotal, runsTotal, sandboxActive, sandboxBundleSizeBytes, sandboxCompletedTotal, sandboxCreatedTotal, sandboxDurationMs, sandboxPatchCount, sandboxTransportDurationMs, schedulerConcurrencyUtilization, schedulerQueueDepth, schedulerWaitDuration, scorerEventsFailed, scorerEventsFinished, scorerEventsStarted, setSmithersLogRunner, smithersMetricCatalog, smithersMetrics, smithersSpanNames, snapshotDuration, snapshotsCaptured, timerDelayDuration, timersCancelled, timersCreated, timersFired, timersPending, toPrometheusMetricName, tokensCacheReadTotal, tokensCacheWriteTotal, tokensContextWindowBucketTotal, tokensContextWindowPerCall, tokensInputPerCall, tokensInputTotal, tokensOutputPerCall, tokensOutputTotal, tokensReasoningTotal, toolCallErrorsTotal, toolCallsTotal, toolDuration, toolOutputTruncatedTotal, trackEvent as trackSmithersEvent, updateCurrentCorrelationContext, updateProcessMetrics, vcsDuration, withCorrelationContext, withCurrentCorrelationContext, withSmithersSpan };
+export { type CorrelationContext, CorrelationContextLive, type CorrelationContextPatch, CorrelationContextService, type CorrelationPatch, type MetricLabels, MetricsService, MetricsServiceLive, type MetricsServiceShape, type MetricsSnapshot, type ResolvedSmithersObservabilityOptions, type RunCancellationSource, type SmithersEvent, type SmithersLogFormat, type SmithersMetricDefinition, SmithersObservability, type SmithersObservabilityOptions, type SmithersObservabilityService, TracingService, TracingServiceLive, activeNodes, activeRuns, annotateSmithersTrace, approvalPending, approvalWaitDuration, approvalsDenied, approvalsGranted, approvalsRequested, attemptDuration, cacheHits, cacheMisses, correlationContextFiberRef, correlationContextToLogAnnotations, createSmithersObservabilityLayer, createSmithersOtelLayer, createSmithersRuntimeLayer, dbQueryDuration, dbRetries, dbTransactionDuration, dbTransactionRetries, dbTransactionRollbacks, errorsTotal, eventsEmittedTotal, externalWaitAsyncPending, getCurrentCorrelationContext, getCurrentCorrelationContextEffect, getCurrentSmithersTraceAnnotations, getCurrentSmithersTraceSpan, hotReloadDuration, hotReloadFailures, hotReloads, httpRequestDuration, httpRequests, logDebug, logError, logInfo, logWarning, makeSmithersSpanAttributes, mergeCorrelationContext, metricsServiceAdapter, nodeDuration, nodeRetriesTotal, nodesFailed, nodesFinished, nodesStarted, processHeapUsedBytes, processMemoryRssBytes, processUptimeSeconds, prometheusContentType, promptSizeBytes, renderPrometheusMetrics, replaysStarted, resolveSmithersObservabilityOptions, responseSizeBytes, rewindDurationMs, rewindFramesDeleted, rewindRollbackTotal, rewindSandboxesReverted, rewindTotal, runDuration, runForksCreated, runWithCorrelationContext, runWithSmithersLogRunner, runsAncestryDepth, runsCancelledTotal, runsCarriedStateBytes, runsContinuedTotal, runsFailedTotal, runsFinishedTotal, runsResumedTotal, runsTotal, sandboxActive, sandboxBundleSizeBytes, sandboxCompletedTotal, sandboxCreatedTotal, sandboxDurationMs, sandboxPatchCount, sandboxTransportDurationMs, schedulerConcurrencyUtilization, schedulerQueueDepth, schedulerWaitDuration, scorerEventsFailed, scorerEventsFinished, scorerEventsStarted, setSmithersLogRunner, smithersMetricCatalog, smithersMetrics, smithersSpanNames, snapshotDuration, snapshotsCaptured, timerDelayDuration, timersCancelled, timersCreated, timersFired, timersPending, toPrometheusMetricName, tokensCacheReadTotal, tokensCacheWriteTotal, tokensContextWindowBucketTotal, tokensContextWindowPerCall, tokensInputPerCall, tokensInputTotal, tokensOutputPerCall, tokensOutputTotal, tokensReasoningTotal, toolCallErrorsTotal, toolCallsTotal, toolDuration, toolOutputTruncatedTotal, trackEvent as trackSmithersEvent, updateCurrentCorrelationContext, updateProcessMetrics, vcsDuration, withCorrelationContext, withCurrentCorrelationContext, withSmithersSpan };

package/src/index.js

@@ -5,6 +5,7 @@
 /** @typedef {import("./_coreMetricsShape.ts").MetricsServiceShape} MetricsServiceShape */
 /** @typedef {import("./_coreMetricsShape.ts").MetricsSnapshot} MetricsSnapshot */
 /** @typedef {import("./ResolvedSmithersObservabilityOptions.ts").ResolvedSmithersObservabilityOptions} ResolvedSmithersObservabilityOptions */
+/** @typedef {import("./SmithersEvent.ts").RunCancellationSource} RunCancellationSource */
 /** @typedef {import("./SmithersEvent.ts").SmithersEvent} SmithersEvent */
 /** @typedef {import("./SmithersLogFormat.ts").SmithersLogFormat} SmithersLogFormat */
 /** @typedef {import("./SmithersMetricDefinition.ts").SmithersMetricDefinition} SmithersMetricDefinition */
@@ -133,4 +134,4 @@ export {
   withCurrentCorrelationContext,
 } from "./correlation.js";
 export { updateCurrentCorrelationContext } from "./correlation.js";
-export { logDebug, logInfo, logWarning, logError, setSmithersLogRunner } from "./logging.js";
+export { logDebug, logInfo, logWarning, logError, runWithSmithersLogRunner, setSmithersLogRunner } from "./logging.js";

package/src/logging.js

@@ -1,4 +1,5 @@
 import { Effect, Logger, LogLevel } from "effect";
+import { AsyncLocalStorage } from "node:async_hooks";
 import { getCurrentSmithersTraceAnnotations } from "./getCurrentSmithersTraceAnnotations.js";
 import {
   correlationContextToLogAnnotations,
@@ -17,6 +18,7 @@ const LOG_LEVEL_WARNING = 3;
 const LOG_LEVEL_ERROR = 4;
 const LOG_RUNNER_KEY = Symbol.for("smithers.observability.logRunner");
 const defaultLoggerLayer = Logger.layer([Logger.consoleLogFmt]);
+const runnerStorage = new AsyncLocalStorage();
 
 /**
  * Resolve the minimum imperative log level. The environment value is an
@@ -77,7 +79,21 @@ const defaultRunner = {
 
 /** @returns {SmithersLogRunner} */
 function getLogRunner() {
-  return getRunnerState().runner ?? defaultRunner;
+  return runnerStorage.getStore() ?? getRunnerState().runner ?? defaultRunner;
+}
+
+/**
+ * Scope imperative Smithers logs to one host-owned Effect runner. Unlike
+ * {@link setSmithersLogRunner}, this is concurrency-safe: parallel embedded
+ * engine instances keep their loggers isolated across awaits and timers.
+ *
+ * @template T
+ * @param {SmithersLogRunner} runner
+ * @param {() => T} execute
+ * @returns {T}
+ */
+export function runWithSmithersLogRunner(runner, execute) {
+  return runnerStorage.run(runner, execute);
 }
 
 /**

package/src/metrics/index.d.ts

@@ -120,7 +120,7 @@ type AgentSessionTranscriptEvent = {
 };
 
 type RunStatus = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "waiting-quota" | "paused" | "finished" | "continued" | "failed" | "cancelled";
-type RunState = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "waiting-quota" | "paused" | "recovering" | "stale" | "orphaned" | "failed" | "cancelled" | "succeeded" | "unknown";
+type RunState = "running" | "waiting-approval" | "waiting-event" | "waiting-timer" | "waiting-quota" | "paused" | "recovering" | "stale" | "orphaned" | "failed" | "cancelled" | "succeeded" | "succeeded-with-failures" | "unknown";
 type AgentCliActionKind = "turn" | "command" | "tool" | "file_change" | "web_search" | "todo_list" | "reasoning" | "warning" | "note";
 type AgentCliActionPhase = "started" | "updated" | "completed";
 type AgentCliEventLevel = "debug" | "info" | "warning" | "error";
@@ -156,6 +156,14 @@ type AgentCliCompletedEvent = {
     usage?: Record<string, unknown>;
 };
 type AgentCliEvent = AgentCliStartedEvent | AgentCliActionEvent | AgentCliCompletedEvent;
+type RunCancellationSource$1 = {
+    kind: "signal" | "rpc" | "cli" | "engine";
+    detail?: string;
+    signal?: string;
+    clientPid?: number;
+    requestId?: string;
+    clientIdentity?: string;
+};
 type SmithersEvent$2 = {
     type: "SupervisorStarted";
     runId: string;
@@ -196,6 +204,19 @@ type SmithersEvent$2 = {
     before: RunState;
     after: RunState;
     timestampMs: number;
+} | {
+    type: "RunConcurrencySaturated";
+    runId: string;
+    requestedDemand: number;
+    effectiveCap: number;
+    remediationCommand: string;
+    /** Present when a lifecycle-linked descendant caused the owner's budget to saturate. */
+    descendantRunId?: string;
+    /** Which owner budget bound admission. Historical events omit this. */
+    budget?: "run" | "subtree";
+    /** Present when `budget` is `subtree`. */
+    subtreeGroupId?: string;
+    timestampMs: number;
 } | {
     type: "RunFinished";
     runId: string;
@@ -217,6 +238,8 @@ type SmithersEvent$2 = {
     type: "RunCancelled";
     runId: string;
     timestampMs: number;
+    /** Missing only on historical or unattributed cancellation events. */
+    source?: RunCancellationSource$1;
 } | {
     type: "RunContinuedAsNew";
     runId: string;
@@ -420,6 +443,16 @@ type SmithersEvent$2 = {
     attempt: number;
     error: unknown;
     timestampMs: number;
+} | {
+    type: "NodeStalled";
+    runId: string;
+    nodeId: string;
+    iteration: number;
+    attempt: number;
+    identicalFailures: number;
+    signature: string;
+    error: unknown;
+    timestampMs: number;
 } | {
     type: "NodeCancelled";
     runId: string;
@@ -706,10 +739,14 @@ type SmithersEvent$2 = {
     model: string;
     agent: string;
     inputTokens: number;
+    /** Non-cached input tokens. Falls back to inputTokens when the provider omits the breakdown. */
+    freshInputTokens?: number;
     outputTokens: number;
     cacheReadTokens?: number;
     cacheWriteTokens?: number;
     reasoningTokens?: number;
+    /** Estimated USD cost from Smithers' built-in model price table. */
+    costUsd?: number;
     timestampMs: number;
 } | {
     type: "SnapshotCaptured";