@senpi/runtime 3.0.6-dev.skill-manager-v2.20260629194505 → 3.0.9-dev.20260710204324

package/dist/runtime/runtime.js

@@ -12,6 +12,7 @@ import { createRiskGuard, createRiskGuardStub } from "./risk-guard.js";
 import { createStrategyLifecycle } from "./strategy-lifecycle.js";
 // RiskGuardianService removed — gate decisions are now real-time in checkGate
 import { createRiskAuditLog } from "../risk/risk-audit-log.js";
+import { createEventStore, registerEventSink, unregisterEventSink, flushEventSink } from "../telemetry/event-store.js";
 import { createNotificationService } from "./notifications.js";
 import { resolveStrategyConfig } from "./map-strategies.js";
 import { mapRuntimeExitToDslConfig } from "./map-exit-config.js";
@@ -31,16 +32,32 @@ import { createActionNotificationSubscriber, isActionLifecycleNotificationsEnabl
 import { MaxLeverageCache } from "../leverage/max-leverage-cache.js";
 import { buildStateLayout } from "./state-layout.js";
 import { createLogger } from "../utils/logger.js";
-import { createTrackingHandler, EventTracker, BatchingTransport, NoOpTransport, TRACKING_API_BASE_URL, PositionIdRegistry, positionRegistryKey, buildRehydrateItems, } from "../tracking/index.js";
+import { buildScannerSummaryEvent, buildRuntimeStartedEvent, buildRuntimeStoppedEvent, buildRuntimeErrorEvent, buildQueueBackpressureEvent, } from "./lifecycle-events.js";
+import { emitEvent } from "../actions/event-derivation.js";
+import { createDomainEventSubscriber } from "./domain-events.js";
+import { PositionIdRegistry, buildRehydrateItems, } from "./position-id-registry.js";
 import { VERSION } from "../version.js";
 import { randomUUID } from "node:crypto";
-import { trackingContext } from "../tracking/context.js";
-import { buildIdentity, createRecordingComponents, createRecordingSubscriber, resolveTelemetryEnabled, } from "../recording/wiring/index.js";
+import { trackingContext } from "../telemetry/context.js";
+import { buildIdentity, } from "../telemetry/identity.js";
 import { runReconcile } from "./reconcile-run.js";
 import { KeyedMutex } from "../utils/keyed-mutex.js";
+import { withSpan } from "../telemetry/tracing.js";
+import { registerDslActivePositionsGauge, registerQueueDepthGauge, recordScannerRun, recordSignalQueueWait, recordRuntimeHeartbeat, } from "../telemetry/metrics.js";
+import { isOtlpEndpointConfigured } from "../telemetry/otel-sdk.js";
+import { identityTraceAttributes, setTraceIdentityAttributes, } from "../telemetry/trace-identity.js";
 const logger = createLogger("[senpi-core][runtime]");
 /** Fixed retry interval for the boot-reconcile loop. */
 const RECONCILE_RETRY_INTERVAL_MS = 15_000;
+/**
+ * Signal-queue depth at which the serial consumer is judged to be falling behind. It processes one
+ * signal at a time (each may run an LLM decision), so a backlog this deep means inflow is outpacing
+ * processing — a `runtime.queue_backpressure` alert. Hardcoded: the depth a stall shows at is a
+ * runtime invariant, not a per-recipe tuning knob.
+ */
+const SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD = 10;
+/** Default liveness-heartbeat window (~15 min) when `health.window_seconds` is unset. */
+const DEFAULT_LIVENESS_WINDOW_SECONDS = 900;
 export class Runtime {
     config;
     options;
@@ -53,9 +70,13 @@ export class Runtime {
     // riskGuardian removed — gate decisions are now real-time in checkGate
     senpi = null;
     healthStateModule = null;
-    trackingHandler = null;
-    recordingSubscriber = null;
-    recordingBatcher = null;
+    domainEventSubscriber = null;
+    /** Wallet under which this runtime's local event sink is registered; cleared on teardown. */
+    eventSinkAddress = null;
+    /** Lazy runtime identity builder; stamps per-trace identity at the signal.process seam. */
+    buildTraceIdentity;
+    traceIdentity;
+    traceIdentityResolved = false;
     started = false;
     startedAt = null;
     /** Serial signal processing: queue and consumer so only one signal is handled at a time. */
@@ -63,13 +84,17 @@ export class Runtime {
     signalQueueWait = null;
     signalConsumerStopped = false;
     signalConsumerDone = null;
+    /** Disposer for this runtime's `runtime.queue_depth` gauge observation; set on start, called on stop. */
+    queueDepthGaugeDispose = null;
+    /** Disposer for this runtime's `dsl.active_positions` gauge observation; set on start, called on stop. */
+    dslActivePositionsGaugeDispose = null;
+    /** Interval emitting this runtime's liveness heartbeat; set on start, cleared on stop. */
+    livenessHeartbeatTimer = null;
     runActionsForSignal;
     strategyConfig;
     actionRuntimeModule;
     /** Stored so we can call off() in stop() and avoid retaining Runtime via scanner EventEmitter. */
     onScannerRunComplete = null;
-    onScannerRunSkip = null;
-    onScannerRunError = null;
     /** Cached strategy id resolved from the runtime wallet on first {@link auditQuery} call. */
     cachedStrategyId = null;
     /** In-flight resolution promise; ensures concurrent first-burst callers reuse one `listStrategies`. */
@@ -137,20 +162,7 @@ export class Runtime {
         if (!this.started) {
             throw new Error("Runtime must be started before accepting external scanner ingests.");
         }
-        console.info("[senpi-runtime] [ingest-trace] runtime.ingestExternalScannerData", JSON.stringify({
-            address: this.strategyConfig.address,
-            scanner: request.scanner,
-            hasAsset: request.asset !== undefined,
-            hasData: request.data !== undefined,
-            hasSignals: request.signals !== undefined,
-            dataKeys: request.data ? Object.keys(request.data) : [],
-        }));
         const result = await this.scannerModule.ingestExternalScannerData(this.strategyConfig.address, request);
-        console.info("[senpi-runtime] [ingest-trace] runtime.ingestExternalScannerData → done", JSON.stringify({
-            accepted: result.accepted,
-            signalCount: result.signalCount,
-            contextUpdated: result.contextUpdated,
-        }));
         return result;
     }
     /**
@@ -309,18 +321,6 @@ export class Runtime {
         validateDslPositionTracking(this.config, hasDsl);
         this.bus = createRuntimeBus();
         this.hookSystem = createHookSystem(this.bus);
-        // Event tracking: passive bus → EventTracker → BatchingTransport POST (requires JWT-capable API key).
-        // Without SENPI_API_KEY, NoOpTransport discards all events; warn once at startup.
-        const trackingApiKey = apiKey ?? process.env.SENPI_AUTH_TOKEN ?? process.env.SENPI_API_KEY;
-        const trackingBaseUrl = process.env.SENPI_TRACKING_URL ?? TRACKING_API_BASE_URL;
-        const trackingTransport = trackingApiKey
-            ? new BatchingTransport({ baseUrl: trackingBaseUrl, apiKey: trackingApiKey })
-            : (() => {
-                logger.warn("event tracking disabled: set SENPI_API_KEY (or pass apiKey) so events reach senpi-agent-telemetry; without it NoOpTransport drops all telemetry");
-                return new NoOpTransport();
-            })();
-        const tracker = new EventTracker(trackingTransport, this.config.name, VERSION, this.config.strategy?.wallet ?? "");
-        this.trackingHandler = createTrackingHandler(this.bus, tracker);
         this.stateManager = new StateManager(stateDir);
         await this.stateManager.hydrate();
         this.positionIdRegistry = new PositionIdRegistry();
@@ -344,41 +344,29 @@ export class Runtime {
         this.senpi = senpi;
         this.strategyConfig = resolveStrategyConfig(this.config);
         await this.rehydratePositionIdRegistry();
-        // Telemetry recording: subscribe the emitter to the bus (on by default; degrades to a noop
-        // emitter when disabled or when no token is present — never a boot failure). Identity + config
-        // fingerprint are built lazily from the unresolved config — only on the live path, and never
-        // from the resolved config (which would inline secrets).
-        const telemetryConfig = this.config.telemetry;
-        const recording = createRecordingComponents({
-            enabled: resolveTelemetryEnabled(telemetryConfig?.enabled, process.env.SENPI_TELEMETRY_ENABLED),
-            token: trackingApiKey,
-            identity: () => buildIdentity({
-                name: this.config.name,
-                recipeVersion: this.config.version,
-                strategyAddress: this.strategyConfig.address,
-                bootId: this.runtimeBootId,
-                unresolvedConfig: this.options.unresolvedConfig,
-            }),
-            scopeVersion: VERSION,
-            baseUrl: telemetryConfig?.url,
-            // v2.0.0: recipe carries flush_interval_seconds; the transport expects ms.
-            flushIntervalMs: typeof telemetryConfig?.flush_interval_seconds === "number"
-                ? telemetryConfig.flush_interval_seconds * 1000
-                : undefined,
+        // The OTLP exporter env is the telemetry master switch: the OTel SDK (stood up at plugin
+        // register()) and the domain-event log both gate on it.
+        const telemetryEnabled = isOtlpEndpointConfigured();
+        // Lazy identity builder. Built only on the live path (never from the resolved
+        // config, which would inline secrets).
+        const buildIdentityEnvelope = () => buildIdentity({
+            name: this.config.name,
+            recipeVersion: this.config.version,
+            strategyAddress: this.strategyConfig.address,
+            group: this.config.group,
+            bootId: this.runtimeBootId,
+            unresolvedConfig: this.options.unresolvedConfig,
         });
-        this.recordingBatcher = recording.batcher;
-        // Only attach the subscriber when telemetry is live (batcher present): with a noop emitter,
-        // per-event record assembly on the serial signal path would be pure dead work.
-        if (recording.batcher) {
-            this.recordingSubscriber = createRecordingSubscriber(this.bus, recording.emitter, {
-                unresolvedConfig: this.options.unresolvedConfig,
-                // Narrow read-only accessor over the registry — recording never sees the
-                // registry or RuntimeContext; sync map read, safe on the bus handler path.
-                resolvePositionId: (address, dex, asset) => this.positionIdRegistry.get(positionRegistryKey(address, dex, asset))?.positionId,
-            });
+        // Share the builder with the signal.process root span (resolved lazily, once per boot).
+        this.buildTraceIdentity = buildIdentityEnvelope;
+        // The event-log half of the real domain events (position.*/dsl.*) → OTel Logs API. Rides the
+        // OTel SDK's log provider. Per-event assembly on the serial signal path is dead work when
+        // telemetry is off, so only attach when enabled.
+        if (telemetryEnabled) {
+            this.domainEventSubscriber = createDomainEventSubscriber(this.bus);
         }
         // Everything from here to `started = true` can throw; without the catch, stop() would
-        // early-return on `!this.started` and the recording batcher/subscriber would leak.
+        // early-return on `!this.started` and the domain-event subscriber would leak.
         try {
             // try {
             //   const { reconcileStateWithBackend } = await import("./reconcile-state.js");
@@ -395,6 +383,10 @@ export class Runtime {
             const riskBlock = this.config.risk;
             const strategyAddress = this.strategyConfig.address;
             const auditLog = riskBlock ? createRiskAuditLog(stateDir) : undefined;
+            // Local domain-event ring. Always on (independent of the OTLP collector) so the agent can query
+            // the trade narrative locally; the singleton logger taps it by wallet via this registration.
+            registerEventSink(strategyAddress, createEventStore(stateDir, strategyAddress));
+            this.eventSinkAddress = strategyAddress;
             // In-memory strategy-lifecycle tracker: owns the self-pause state that
             // RiskGuard used to hold and fires the hook + notification on a running→paused
             // transition. RiskGuard stays stateless and only reports gate evaluations.
@@ -423,7 +415,7 @@ export class Runtime {
                 getDslStatesForStrategy: scannerProviders?.getDslStatesForStrategy
                     ?? (async (address) => this.stateManager.listDslStates(address)),
             };
-            this.scannerModule = createScannerModuleFromRuntimeConfig(this.config, layout.scannerRoot, augmentedProviders);
+            this.scannerModule = createScannerModuleFromRuntimeConfig(this.config, layout.scannerRoot, augmentedProviders, undefined, buildIdentityEnvelope);
             logger.info("Runtime.start() scanner module created, calling start()");
             // Pre-load max leverage cache (non-blocking — log warning on failure)
             const maxLeverageCache = new MaxLeverageCache(senpi);
@@ -493,6 +485,7 @@ export class Runtime {
                     // against the trade loop on the same reconcileMutex key, and can resolve the boot gate.
                     reconcileSerialize: (fn) => this.reconcileMutex.withKeyLock("reconcile", fn),
                     reconcileGateResolve: () => this.resolveReconcileGate(),
+                    identity: buildIdentityEnvelope,
                 });
             }
             else {
@@ -529,34 +522,40 @@ export class Runtime {
                 },
             });
             this.onScannerRunComplete = (event) => {
-                console.info("[senpi-runtime] [ingest-trace] onScannerRunComplete → enqueue", JSON.stringify({
-                    address: event.result.address,
-                    scannerId: event.result.scannerId,
-                    signalCount: Array.isArray(event.result.signals) ? event.result.signals.length : 0,
-                    status: event.result.status,
-                    queueDepth: this.signalQueue.length + 1,
-                }));
-                const correlationId = randomUUID();
-                // Telemetry-owned per-scan-run id (NOT the deprecated correlation id): stamped on the
-                // scanner_run emit and echoed through the pass so scan + pass land in one trace.
-                const runId = randomUUID();
-                this.signalQueue.push({ result: event.result, correlationId, enqueuedAt: Date.now(), runId });
+                // Per-scan-run id: stamped on the scanner_run emit and echoed through the pass so scan +
+                // pass land in one trace. Prefer the id minted at the scanner.run span so scan + pass
+                // share one senpi.run.id.
+                const runId = event.result.runId ?? randomUUID();
+                this.signalQueue.push({ result: event.result, enqueuedAt: Date.now(), runId });
+                // Edge-triggered backpressure alarm: depth rises one enqueue at a time, so `=== threshold`
+                // fires exactly once as it crosses up and re-arms only after the consumer drains back below.
+                if (this.signalQueue.length === SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD) {
+                    emitEvent(logger, () => buildQueueBackpressureEvent({
+                        depth: this.signalQueue.length,
+                        threshold: SIGNAL_QUEUE_BACKPRESSURE_THRESHOLD,
+                        scannerId: event.result.scannerId,
+                        address: this.strategyConfig.address,
+                    }));
+                }
                 if (this.signalQueueWait) {
                     const w = this.signalQueueWait;
                     this.signalQueueWait = null;
                     w();
                 }
-                // Emit scanner run for tracking (startedAt is data stamping for telemetry durations)
-                this.bus.emit("tracking.scanner_run", {
+                // Scanner-run summary payload (startedAt is data stamping for telemetry durations).
+                const scannerRunPayload = {
                     address: event.result.address,
                     data: {
                         ...event.result,
                         startedAt: event.startedAt,
                         runId,
-                        __correlationId: correlationId,
                     },
                     timestamp: new Date().toISOString(),
-                });
+                };
+                // First-class event log. The build runs inside emitEvent's guard — telemetry never
+                // disturbs the scan path.
+                emitEvent(logger, () => buildScannerSummaryEvent(scannerRunPayload, Date.now()));
+                recordScannerRun({ scannerId: event.result.scannerId, outcome: event.result.status });
                 // Forward scanner summary alerts (e.g. watchdog) to notification service
                 const alerts = event.result.summary?.alerts;
                 if (Array.isArray(alerts) && alerts.length > 0) {
@@ -566,37 +565,6 @@ export class Runtime {
                 }
             };
             this.scannerModule.on("scanner:run:complete", this.onScannerRunComplete);
-            // Surface skipped/errored scanner runs on the bus for telemetry (add-only observation events —
-            // distinct names so existing `tracking.scanner_run` consumers see an unchanged payload mix).
-            // Neither enqueues a signal: only completed runs feed the signal queue.
-            this.onScannerRunSkip = (event) => {
-                this.bus.emit("tracking.scanner_run_skipped", {
-                    address: event.address,
-                    data: {
-                        scannerId: event.scannerId,
-                        status: event.status,
-                        reason: event.reason,
-                        timestamp: event.timestamp,
-                        startedAt: event.startedAt,
-                    },
-                    timestamp: new Date().toISOString(),
-                });
-            };
-            this.onScannerRunError = (event) => {
-                this.bus.emit("tracking.scanner_run_error", {
-                    address: event.address,
-                    data: {
-                        scannerId: event.scannerId,
-                        status: "error",
-                        reason: typeof event.error === "string" ? event.error : String(event.error),
-                        timestamp: event.timestamp,
-                        startedAt: event.startedAt,
-                    },
-                    timestamp: new Date().toISOString(),
-                });
-            };
-            this.scannerModule.on("scanner:run:skip", this.onScannerRunSkip);
-            this.scannerModule.on("scanner:run:error", this.onScannerRunError);
             this.signalConsumerDone = this.runSignalConsumer();
             logger.info("Runtime.start() invoking scannerModule.start()");
             await this.scannerModule.start();
@@ -640,14 +608,6 @@ export class Runtime {
                     this.scannerModule.off("scanner:run:complete", this.onScannerRunComplete);
                     this.onScannerRunComplete = null;
                 }
-                if (this.onScannerRunSkip != null) {
-                    this.scannerModule.off("scanner:run:skip", this.onScannerRunSkip);
-                    this.onScannerRunSkip = null;
-                }
-                if (this.onScannerRunError != null) {
-                    this.scannerModule.off("scanner:run:error", this.onScannerRunError);
-                    this.onScannerRunError = null;
-                }
                 if (this.actionNotificationSubscriber != null) {
                     this.actionNotificationSubscriber.dispose();
                     this.actionNotificationSubscriber = null;
@@ -664,22 +624,66 @@ export class Runtime {
             catch (cleanupErr) {
                 logger.warn("[runtime] partial-state cleanup error on failed start (ignored)", { cleanupErr });
             }
-            // Recording-only cleanup (telemetry must not leak on a failed boot); the error rethrows.
-            await this.disposeRecording();
+            // Telemetry cleanup (must not leak on a failed boot); the error rethrows.
+            await this.disposeSubscribers();
             throw err;
         }
-        this.bus.emit("tracking.runtime_started", {
+        // First-class event log. Carries the UNRESOLVED config hash (the form `computeConfigHash` uses),
+        // so the printed hash matches `senpi.config.hash` on the spans/logs. The build runs inside
+        // emitEvent's guard — telemetry never blocks boot.
+        emitEvent(logger, () => buildRuntimeStartedEvent({
             address: this.strategyConfig.address,
-            data: {
-                config: this.config,
-                strategyAddress: this.strategyConfig.address,
-                runtimeVersion: VERSION,
-                hasDsl: this.dslPlugin != null,
-                scannerCount: this.config.scanners?.length ?? 0,
-                actionCount: this.config.actions?.length ?? 0,
-            },
-            timestamp: new Date().toISOString(),
+            recipeName: this.config.name,
+            recipeVersion: this.config.version,
+            runtimeVersion: VERSION,
+            scannerCount: this.config.scanners?.length ?? 0,
+            actionCount: this.config.actions?.length ?? 0,
+            hasDsl: this.dslPlugin != null,
+            unresolvedConfig: this.options.unresolvedConfig,
+            slots: this.strategyConfig.slots,
+            tradingRisk: this.strategyConfig.tradingRisk,
+            marginPerSlot: this.strategyConfig.marginPerSlot,
+            marginPct: this.strategyConfig.marginPct,
+            defaultLeverage: this.strategyConfig.defaultLeverage,
+        }));
+        // Observable gauge of the serial signal-queue depth — its absence/stall is alertable without a
+        // trace or log scan. Reads the live depth at each collection; disposed in stop().
+        const walletDim = this.strategyConfig.address.toLowerCase();
+        const group = this.config.group;
+        this.queueDepthGaugeDispose = registerQueueDepthGauge({
+            strategyAddress: walletDim,
+            group,
+            read: () => this.getSignalQueueDepth(),
         });
+        // Liveness heartbeat: a positive tick every window, dimensioned by wallet/group. A dead runtime
+        // can't emit a "down" event — only the absence of these ticks detects it. Fire one now so a
+        // runtime that dies before its first window still leaves a mark. Unref'd so it never holds the
+        // process open; cleared in stop().
+        const windowSec = this.config.health?.window_seconds ?? DEFAULT_LIVENESS_WINDOW_SECONDS;
+        recordRuntimeHeartbeat({ strategyAddress: walletDim, group });
+        this.livenessHeartbeatTimer = setInterval(() => recordRuntimeHeartbeat({ strategyAddress: walletDim, group }), windowSec * 1000);
+        this.livenessHeartbeatTimer.unref?.();
+        // Observable gauge of active DSL-managed positions, split by exit phase. Only meaningful when this
+        // runtime runs the exit engine; counts this wallet's positions live at each collection (no snapshot).
+        if (this.dslPlugin) {
+            this.dslActivePositionsGaugeDispose = registerDslActivePositionsGauge({
+                strategyAddress: walletDim,
+                group,
+                read: () => {
+                    let phase1 = 0;
+                    let phase2 = 0;
+                    for (const p of this.dslPlugin?.listActivePositions() ?? []) {
+                        if (p.address?.toLowerCase() !== walletDim)
+                            continue;
+                        if (p.phase === 2)
+                            phase2++;
+                        else
+                            phase1++;
+                    }
+                    return { phase1, phase2 };
+                },
+            });
+        }
     }
     /**
      * Cancel exchange SLs and archive DSL state for the strategy, then log removal.
@@ -695,7 +699,24 @@ export class Runtime {
     }
     /** Tear down DSL for the runtime strategy before deleting the runtime (gateway / CLI). */
     async teardownDslForRuntimeDeletion() {
-        return this.removeStrategy(this.strategyConfig.address);
+        await this.removeStrategy(this.strategyConfig.address);
+    }
+    /**
+     * Resolve the per-boot trace identity once. The envelope is immutable per boot, so build it
+     * lazily and cache it. The build is guarded: a failing identity builder must never abort
+     * signal processing.
+     */
+    resolveTraceIdentity() {
+        if (!this.traceIdentityResolved) {
+            this.traceIdentityResolved = true;
+            try {
+                this.traceIdentity = this.buildTraceIdentity?.();
+            }
+            catch (err) {
+                logger.warn("[runtime] failed to build trace identity; continuing without it", { err });
+            }
+        }
+        return this.traceIdentity;
     }
     /**
      * Single consumer: process one scanner:run:complete at a time so slot counts stay correct.
@@ -714,84 +735,113 @@ export class Runtime {
             if (!item)
                 continue;
             // Pin tracking context for this signal flow so LLM spans (emitted by
-            // OpenClawLlmDecision / AnthropicLlmDecision) can stamp correlation_id,
+            // OpenClawLlmDecision / AnthropicLlmDecision) can stamp run id,
             // address, recipe_name, runtime_id, and runtime_version automatically.
             const queuedItem = item;
+            // Resolved once here so the same identity rides both the trace (via the span stamp)
+            // and the logs (via the context bag below) — one source, no drift.
+            const identity = this.resolveTraceIdentity();
             await trackingContext.run({
-                correlationId: queuedItem.correlationId,
+                runId: queuedItem.runId,
                 address: queuedItem.result.address,
                 recipeName: this.config.name,
                 runtimeId: this.options.runtimeId ?? "",
                 runtimeVersion: VERSION,
                 runtimeBootId: this.runtimeBootId,
+                identityAttributes: identity
+                    ? identityTraceAttributes(identity)
+                    : undefined,
             }, async () => {
-                try {
-                    // Block until the first boot reconcile succeeds (or runtime is stopped).
-                    await this.reconcileGate?.promise;
-                    // If stop() resolved the gate during shutdown, do NOT trade and do NOT
-                    // contend for reconcileMutex — a boot reconcile that is still in-flight
-                    // (or stuck) holds that key, and blocking here would hang stop()'s
-                    // `await signalConsumerDone`. Bail before the mutex acquire.
-                    if (this.signalConsumerStopped) {
-                        return;
-                    }
-                    // Evict the blackout backlog. Items enqueued before the gate
-                    // resolved were formed while the runtime could not confirm its own
-                    // positions (unreconciled / exchange-flaky), so acting on them now is
-                    // stale — the scanner re-emits fresh signals once reconciled. Strict
-                    // `<`: items enqueued at/after gate-open are processed normally. This
-                    // is gate-scoped, NOT a queue TTL — nothing is evicted once the
-                    // backlog is cleared.
-                    if (this.gateResolvedAt != null && queuedItem.enqueuedAt < this.gateResolvedAt) {
-                        this.staleSignalsEvicted++;
-                        logger.warn("[runtime] evicting stale pre-reconcile signal (enqueued before gate open)", {
-                            address: queuedItem.result.address,
-                            scannerId: queuedItem.result.scannerId,
-                            enqueuedAt: queuedItem.enqueuedAt,
-                            gateResolvedAt: this.gateResolvedAt,
-                            totalEvicted: this.staleSignalsEvicted,
-                        });
-                        return;
-                    }
-                    console.info("[senpi-runtime] [ingest-trace] consumer dequeue → runActionsForSignal", JSON.stringify({
-                        address: queuedItem.result.address,
-                        scannerId: queuedItem.result.scannerId,
-                        signalCount: Array.isArray(queuedItem.result.signals) ? queuedItem.result.signals.length : 0,
-                        queueRemaining: this.signalQueue.length,
-                    }));
-                    const usedSlotsByStrategy = await this.stateManager.getUsedSlotsByStrategy([this.strategyConfig.address]);
-                    const payload = buildActionScanPayload(queuedItem.result, this.strategyConfig, {
-                        usedSlotsByStrategy,
-                        address: queuedItem.result.address,
-                    });
-                    const hookEvent = {
-                        type: HOOK_EVENT.ON_SIGNAL_DETECTED,
-                        runtimeName: this.config.name,
-                        address: queuedItem.result.address,
-                        data: {
-                            ...payload,
-                            __correlationId: queuedItem.correlationId,
-                            __scanRunId: queuedItem.runId,
-                        },
-                        timestamp: new Date().toISOString(),
-                    };
-                    this.bus.emit(HOOK_EVENT.ON_SIGNAL_DETECTED, hookEvent);
-                    // Serialize trade loop vs reconcile — reconcileMutex outer, key-lock inner.
-                    await this.reconcileMutex.withKeyLock("reconcile", () => this.runActionsForSignal(hookEvent));
-                    console.info("[senpi-runtime] [ingest-trace] runActionsForSignal complete", JSON.stringify({ address: queuedItem.result.address, scannerId: queuedItem.result.scannerId }));
+                // Block until the first boot reconcile succeeds (or runtime is stopped).
+                await this.reconcileGate?.promise;
+                // If stop() resolved the gate during shutdown, do NOT trade and do NOT
+                // contend for reconcileMutex — a boot reconcile that is still in-flight
+                // (or stuck) holds that key, and blocking here would hang stop()'s
+                // `await signalConsumerDone`. Bail before the mutex acquire.
+                if (this.signalConsumerStopped) {
+                    return;
                 }
-                catch (err) {
-                    const message = err instanceof Error ? err.message : String(err);
-                    const stack = err instanceof Error ? err.stack : undefined;
-                    console.error("[senpi-runtime] signal processing error", message, stack ?? "");
-                    this.bus.emit("signal_processing_error", {
-                        error: err,
+                // Evict the blackout backlog. Items enqueued before the gate
+                // resolved were formed while the runtime could not confirm its own
+                // positions (unreconciled / exchange-flaky), so acting on them now is
+                // stale — the scanner re-emits fresh signals once reconciled. Strict
+                // `<`: items enqueued at/after gate-open are processed normally. This
+                // is gate-scoped, NOT a queue TTL — nothing is evicted once the
+                // backlog is cleared.
+                if (this.gateResolvedAt != null && queuedItem.enqueuedAt < this.gateResolvedAt) {
+                    this.staleSignalsEvicted++;
+                    logger.warn("[runtime] evicting stale pre-reconcile signal (enqueued before gate open)", {
                         address: queuedItem.result.address,
                         scannerId: queuedItem.result.scannerId,
-                        runId: queuedItem.runId,
-                        timestamp: new Date().toISOString(),
+                        enqueuedAt: queuedItem.enqueuedAt,
+                        gateResolvedAt: this.gateResolvedAt,
+                        totalEvicted: this.staleSignalsEvicted,
                     });
+                    return;
+                }
+                // Root span born at dequeue; spans for the action handlers nest beneath it.
+                // Identity (incl. strategy address) rides every span via the per-trace stamp below;
+                // run.id is the join key back to the producing scanner run.
+                const spanAttrs = {};
+                if (queuedItem.runId) {
+                    spanAttrs["senpi.run.id"] = queuedItem.runId;
+                }
+                // Carry the producing scanner id so the signal span joins/filters by scanner,
+                // matching the key the sibling scanner.run span stamps.
+                if (queuedItem.result.scannerId) {
+                    spanAttrs["senpi.scanner.id"] = queuedItem.result.scannerId;
                 }
+                await withSpan("signal.process", spanAttrs, async (span) => {
+                    if (identity)
+                        await setTraceIdentityAttributes(identity);
+                    // Serial FIFO: time spent waiting behind earlier signals, and the depth still
+                    // behind this one — the backpressure a per-pass duration alone can't show.
+                    // Measured from enqueue, but the pre-gate backlog is evicted above before the
+                    // span opens, so every signal that reaches here was enqueued after the (one-shot)
+                    // reconcile gate resolved — i.e. this is pure FIFO wait, not gate wait.
+                    const queueWaitMs = Date.now() - queuedItem.enqueuedAt;
+                    span.setAttribute("senpi.queue.wait_ms", queueWaitMs);
+                    span.setAttribute("senpi.queue.depth", this.signalQueue.length);
+                    recordSignalQueueWait(queueWaitMs / 1000, { scannerId: queuedItem.result.scannerId });
+                    try {
+                        const usedSlotsByStrategy = await this.stateManager.getUsedSlotsByStrategy([this.strategyConfig.address]);
+                        const payload = buildActionScanPayload(queuedItem.result, this.strategyConfig, {
+                            usedSlotsByStrategy,
+                            address: queuedItem.result.address,
+                        });
+                        const hookEvent = {
+                            type: HOOK_EVENT.ON_SIGNAL_DETECTED,
+                            runtimeName: this.config.name,
+                            address: queuedItem.result.address,
+                            data: {
+                                ...payload,
+                                __scanRunId: queuedItem.runId,
+                            },
+                            timestamp: new Date().toISOString(),
+                        };
+                        this.bus.emit(HOOK_EVENT.ON_SIGNAL_DETECTED, hookEvent);
+                        // Serialize trade loop vs reconcile — reconcileMutex outer, key-lock inner.
+                        await this.reconcileMutex.withKeyLock("reconcile", () => this.runActionsForSignal(hookEvent));
+                        console.info("[senpi-runtime] [ingest-trace] runActionsForSignal complete", JSON.stringify({ address: queuedItem.result.address, scannerId: queuedItem.result.scannerId }));
+                    }
+                    catch (err) {
+                        const message = err instanceof Error ? err.message : String(err);
+                        // Structured error log: normalizes to exception.* attributes and, since the catch
+                        // swallows the error (so withSpan's auto-record never fires), records it on the active
+                        // span + sets ERROR status. Replaces the prior console.error + manual span calls.
+                        logger.error("signal processing error", err);
+                        this.bus.emit("signal_processing_error", {
+                            error: err,
+                            address: queuedItem.result.address,
+                            scannerId: queuedItem.result.scannerId,
+                            runId: queuedItem.runId,
+                            timestamp: new Date().toISOString(),
+                        });
+                        // First-class event log alongside the legacy bus emit. The build runs inside
+                        // emitEvent's guard — telemetry never disturbs the consumer.
+                        emitEvent(logger, () => buildRuntimeErrorEvent("signal_processing", queuedItem.result.scannerId, message));
+                    }
+                });
             });
         }
     }
@@ -852,23 +902,6 @@ export class Runtime {
             }
             const durationMs = Date.now() - t0;
             const { adopted, resized, archived } = result.counts;
-            // Telemetry must never break reconcile control flow. bus.emit is a Node
-            // EventEmitter: a listener that throws synchronously makes emit re-throw,
-            // which would reject this loop and leave the gate forever closed. Contain it.
-            try {
-                this.bus.emit("tracking.runtime_reconciled", {
-                    address,
-                    data: { adopted, resized, archived, success: result.success, attempt, durationMs },
-                    timestamp: new Date().toISOString(),
-                });
-            }
-            catch (err) {
-                logger.warn("[runtime][reconcile] tracking.runtime_reconciled listener threw; contained", {
-                    address,
-                    attempt,
-                    err,
-                });
-            }
             if (result.success) {
                 logger.info("[runtime][reconcile] boot reconcile succeeded", { address, attempt, adopted, resized, archived });
                 this.resolveReconcileGate();
@@ -887,17 +920,20 @@ export class Runtime {
         gate.resolve();
     }
     /**
-     * Dispose the recording subscriber, then flush + stop the batcher (best-effort). Idempotent;
-     * shared by stop() and the failed-start cleanup path in start().
+     * Dispose the domain-event subscriber. Idempotent; shared by stop() and the failed-start
+     * cleanup path in start().
      */
-    async disposeRecording() {
-        if (this.recordingSubscriber) {
-            this.recordingSubscriber.dispose();
-            this.recordingSubscriber = null;
+    async disposeSubscribers() {
+        if (this.domainEventSubscriber) {
+            this.domainEventSubscriber.dispose();
+            this.domainEventSubscriber = null;
         }
-        if (this.recordingBatcher) {
-            await this.recordingBatcher.shutdown().catch(() => { });
-            this.recordingBatcher = null;
+        if (this.eventSinkAddress !== null) {
+            // Drain the sink's queued fire-and-forget appends before dropping it, so a graceful stop
+            // doesn't discard events that were enqueued but not yet written.
+            await flushEventSink(this.eventSinkAddress);
+            unregisterEventSink(this.eventSinkAddress);
+            this.eventSinkAddress = null;
         }
     }
     async stop() {
@@ -912,15 +948,21 @@ export class Runtime {
         }
         // Resolve the gate so the consumer doesn't hang waiting for a reconcile that won't come.
         this.reconcileGate?.resolve();
-        this.bus.emit("tracking.runtime_stopped", {
-            address: this.strategyConfig.address,
-            data: {
-                uptimeMs: Date.now() - (this.startedAt ?? Date.now()),
-                runtimeVersion: VERSION,
-                strategyAddress: this.strategyConfig.address,
-            },
-            timestamp: new Date().toISOString(),
-        });
+        const uptimeMs = Date.now() - (this.startedAt ?? Date.now());
+        // First-class event log. The build runs inside emitEvent's guard — telemetry never blocks shutdown.
+        emitEvent(logger, () => buildRuntimeStoppedEvent(VERSION, uptimeMs, this.strategyConfig.address));
+        if (this.queueDepthGaugeDispose) {
+            this.queueDepthGaugeDispose();
+            this.queueDepthGaugeDispose = null;
+        }
+        if (this.dslActivePositionsGaugeDispose) {
+            this.dslActivePositionsGaugeDispose();
+            this.dslActivePositionsGaugeDispose = null;
+        }
+        if (this.livenessHeartbeatTimer) {
+            clearInterval(this.livenessHeartbeatTimer);
+            this.livenessHeartbeatTimer = null;
+        }
         this.signalConsumerStopped = true;
         if (this.signalQueueWait) {
             const w = this.signalQueueWait;
@@ -936,14 +978,6 @@ export class Runtime {
             this.scannerModule.off("scanner:run:complete", this.onScannerRunComplete);
             this.onScannerRunComplete = null;
         }
-        if (this.onScannerRunSkip != null) {
-            this.scannerModule.off("scanner:run:skip", this.onScannerRunSkip);
-            this.onScannerRunSkip = null;
-        }
-        if (this.onScannerRunError != null) {
-            this.scannerModule.off("scanner:run:error", this.onScannerRunError);
-            this.onScannerRunError = null;
-        }
         if (this.actionNotificationSubscriber != null) {
             this.actionNotificationSubscriber.dispose();
             this.actionNotificationSubscriber = null;
@@ -958,13 +992,7 @@ export class Runtime {
             this.senpi = null;
         }
         this.healthStateModule = null;
-        if (this.trackingHandler) {
-            await this.trackingHandler.dispose();
-            this.trackingHandler = null;
-        }
-        // Telemetry recording teardown: stop the subscriber, then flush + stop the batcher (best-effort)
-        // so the just-recorded runtime_stopped row ships before the process exits.
-        await this.disposeRecording();
+        await this.disposeSubscribers();
         if (this.bus.removeAllListeners) {
             this.bus.removeAllListeners();
         }