openclaw-observability 2026.3.21 → 2026.3.23

package/dist/index.js

@@ -6,13 +6,46 @@
  * OpenClaw hooks:
  *   Agent: before_model_resolve, before_prompt_build, before_agent_start, agent_end
  *   LLM:   llm_input, llm_output
- *   Tool:  before_tool_call, after_tool_call, tool_result_persist
+ *   Tool:  before_tool_call, after_tool_call
  *   Msg:   message_received, message_sending, message_sent, before_message_write
  *   Ctx:   before_compaction, after_compaction, before_reset
  *   Session: session_start, session_end
  *   Subagent: subagent_spawning, subagent_delivery_target, subagent_spawned, subagent_ended
  *   Gateway: gateway_start, gateway_stop
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.activate = activate;
 const config_1 = require("./config");
@@ -23,6 +56,10 @@ const duckdb_local_writer_1 = require("./storage/duckdb-local-writer");
 const buffer_1 = require("./storage/buffer");
 const routes_1 = require("./web/routes");
 const scanner_1 = require("./security/scanner");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const os = __importStar(require("node:os"));
+const node_crypto_1 = require("node:crypto");
 /* ------------------------------------------------------------------ */
 /*  Runtime state                                                       */
 /* ------------------------------------------------------------------ */
@@ -32,8 +69,16 @@ const MAX_MAP_ENTRIES = 1000;
 const MAP_ENTRY_TTL_MS = 300_000;
 /** Max alert buffer size */
 const MAX_ALERT_BUFFER = 500;
-/** Track start time of each LLM/tool call */
-const runStartTimes = new Map();
+/** Track start time of each LLM call (runId -> timestamp) */
+const llmRunStartTimes = new Map();
+/** Resolve runtime event stream records back to observability sessions */
+const runSessionIds = new Map();
+/** Track start times of tool calls (supports same-key concurrency via FIFO queue) */
+const toolCallStartTimes = new Map();
+const pendingRuntimeEvents = new Map();
+const runtimeTextSeen = new Set();
+const replayedAssistantMessages = new Map();
+const sessionTranscriptPathCache = new Map();
 /**
  * Map: requestUrl → usage from SSE final chunk
  * We key by a monotonically increasing request ID since we can't know
@@ -41,15 +86,242 @@ const runStartTimes = new Map();
  */
 const sseUsageCache = new Map();
 let sseRequestCounter = 0;
-/**
- * Most recent SSE usage entry (LIFO — the llm_output hook fires shortly
- * after the stream completes, so the latest entry is the one we want)
- */
-let latestSseUsage = null;
 /** Original fetch reference */
 let originalFetch = null;
 /** Whether the fetch interceptor is installed */
 let fetchInterceptorInstalled = false;
+function getRuntimeTextSeenKey(runId, stream) {
+    return `${runId}:${stream}`;
+}
+function parseMessageTimestamp(value) {
+    if (typeof value === 'number' && Number.isFinite(value)) {
+        return value > 1e12 ? value : value * 1000;
+    }
+    if (typeof value === 'string') {
+        const parsed = Date.parse(value);
+        return Number.isNaN(parsed) ? null : parsed;
+    }
+    return null;
+}
+function extractTextParts(value) {
+    if (typeof value === 'string') {
+        const trimmed = value.trim();
+        return trimmed ? [trimmed] : [];
+    }
+    if (!Array.isArray(value))
+        return [];
+    const parts = [];
+    for (const item of value) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const block = item;
+        const type = typeof block.type === 'string' ? block.type : '';
+        if (type === 'text') {
+            const text = typeof block.text === 'string' ? block.text.trim() : '';
+            if (text)
+                parts.push(text);
+            continue;
+        }
+        if (type === 'thinking') {
+            const thinking = typeof block.thinking === 'string' ? block.thinking.trim() : '';
+            if (thinking)
+                parts.push(thinking);
+            continue;
+        }
+    }
+    return parts;
+}
+function extractAssistantArtifacts(lastAssistant) {
+    if (!lastAssistant || typeof lastAssistant !== 'object') {
+        return { texts: [], thinking: [] };
+    }
+    const source = lastAssistant;
+    const messageRecord = source.message && typeof source.message === 'object'
+        ? source.message
+        : source;
+    const content = messageRecord.content;
+    if (!Array.isArray(content)) {
+        return {
+            texts: extractTextParts(content),
+            thinking: [],
+        };
+    }
+    const texts = [];
+    const thinking = [];
+    for (const item of content) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const block = item;
+        const type = typeof block.type === 'string' ? block.type : '';
+        if (type === 'text') {
+            const text = typeof block.text === 'string' ? block.text.trim() : '';
+            if (text)
+                texts.push(text);
+            continue;
+        }
+        if (type === 'thinking') {
+            const value = typeof block.thinking === 'string' ? block.thinking.trim() : '';
+            if (value)
+                thinking.push(value);
+        }
+    }
+    return { texts, thinking };
+}
+function extractToolUseAssistantMessage(historyMessages) {
+    for (let i = historyMessages.length - 1; i >= 0; i--) {
+        const item = historyMessages[i];
+        if (!item || typeof item !== 'object')
+            continue;
+        const msg = item;
+        if (msg.role !== 'assistant')
+            continue;
+        const content = Array.isArray(msg.content) ? msg.content : [];
+        if (content.length === 0)
+            continue;
+        const texts = [];
+        const thinking = [];
+        const toolCallIds = [];
+        let hasToolCall = false;
+        for (const blockValue of content) {
+            if (!blockValue || typeof blockValue !== 'object')
+                continue;
+            const block = blockValue;
+            const type = typeof block.type === 'string' ? block.type : '';
+            if (type === 'text') {
+                const text = typeof block.text === 'string' ? block.text.trim() : '';
+                if (text)
+                    texts.push(text);
+                continue;
+            }
+            if (type === 'thinking') {
+                const text = typeof block.thinking === 'string' ? block.thinking.trim() : '';
+                if (text)
+                    thinking.push(text);
+                continue;
+            }
+            if (type === 'toolCall') {
+                hasToolCall = true;
+                if (typeof block.id === 'string' && block.id.trim()) {
+                    toolCallIds.push(block.id);
+                }
+            }
+        }
+        const stopReason = typeof msg.stopReason === 'string' ? msg.stopReason : undefined;
+        if (!hasToolCall && stopReason !== 'toolUse')
+            continue;
+        if (texts.length === 0 && thinking.length === 0)
+            continue;
+        return {
+            messageId: typeof msg.id === 'string' ? msg.id : undefined,
+            timestampMs: parseMessageTimestamp(msg.timestamp),
+            stopReason,
+            texts,
+            thinking,
+            toolCallIds,
+        };
+    }
+    return null;
+}
+function resolveSessionTranscriptPath(sessionId) {
+    if (!sessionId || sessionId === 'unknown')
+        return null;
+    if (sessionTranscriptPathCache.has(sessionId)) {
+        return sessionTranscriptPathCache.get(sessionId) ?? null;
+    }
+    try {
+        const agentsRoot = path.join(os.homedir(), '.openclaw', 'agents');
+        const agentDirs = fs.readdirSync(agentsRoot, { withFileTypes: true });
+        for (const dir of agentDirs) {
+            if (!dir.isDirectory())
+                continue;
+            const candidate = path.join(agentsRoot, dir.name, 'sessions', `${sessionId}.jsonl`);
+            if (fs.existsSync(candidate)) {
+                sessionTranscriptPathCache.set(sessionId, candidate);
+                return candidate;
+            }
+        }
+    }
+    catch {
+        // ignore lookup failures
+    }
+    sessionTranscriptPathCache.set(sessionId, null);
+    return null;
+}
+function extractToolUseAssistantMessagesFromTranscript(params) {
+    const transcriptPath = resolveSessionTranscriptPath(params.sessionId);
+    if (!transcriptPath)
+        return [];
+    const lowerBound = (params.runStartedAt ?? params.runEndedAt - 300_000) - 2000;
+    const upperBound = params.runEndedAt + 2000;
+    let raw = '';
+    try {
+        raw = fs.readFileSync(transcriptPath, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    const results = [];
+    const lines = raw.split('\n');
+    for (const line of lines) {
+        if (!line.trim())
+            continue;
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (record.type !== 'message')
+            continue;
+        const message = record.message;
+        if (!message || message.role !== 'assistant')
+            continue;
+        const timestampMs = parseMessageTimestamp(record.timestamp);
+        if (timestampMs === null || timestampMs < lowerBound || timestampMs > upperBound)
+            continue;
+        const content = Array.isArray(message.content) ? message.content : [];
+        if (content.length === 0)
+            continue;
+        const texts = [];
+        const thinking = [];
+        const toolCallIds = [];
+        let hasToolCall = false;
+        for (const blockValue of content) {
+            if (!blockValue || typeof blockValue !== 'object')
+                continue;
+            const block = blockValue;
+            const type = typeof block.type === 'string' ? block.type : '';
+            if (type === 'text') {
+                const text = typeof block.text === 'string' ? block.text.trim() : '';
+                if (text)
+                    texts.push(text);
+                continue;
+            }
+            if (type === 'thinking') {
+                const text = typeof block.thinking === 'string' ? block.thinking.trim() : '';
+                if (text)
+                    thinking.push(text);
+                continue;
+            }
+            if (type === 'toolCall') {
+                hasToolCall = true;
+                if (typeof block.id === 'string' && block.id.trim()) {
+                    toolCallIds.push(block.id);
+                }
+            }
+        }
+        const stopReason = typeof message.stopReason === 'string' ? message.stopReason : undefined;
+        if (!hasToolCall && stopReason !== 'toolUse')
+            continue;
+        if (texts.length === 0 && thinking.length === 0)
+            continue;
+        const messageId = typeof record.id === 'string' ? record.id : `${timestampMs}:${toolCallIds.join(',')}`;
+        results.push({ messageId, timestampMs, stopReason, texts, thinking, toolCallIds });
+    }
+    results.sort((a, b) => a.timestampMs - b.timestampMs);
+    return results;
+}
 function installFetchInterceptor() {
     if (fetchInterceptorInstalled)
         return;
@@ -142,7 +414,6 @@ function installFetchInterceptor() {
                                 processSseLines();
                             if (extractedUsage) {
                                 sseUsageCache.set(requestId, extractedUsage);
-                                latestSseUsage = extractedUsage;
                             }
                             controller.close();
                             break;
@@ -175,11 +446,28 @@ function uninstallFetchInterceptor() {
     }
 }
 /**
- * Pop the most recent SSE usage (consume once per llm_output call)
+ * Consume one SSE usage entry for a specific run.
+ * Picks the earliest entry whose timestamp is not older than run start
+ * (with small clock-tolerance). Falls back to FIFO oldest entry.
  */
-function popLatestSseUsage() {
-    const usage = latestSseUsage;
-    latestSseUsage = null;
+function consumeSseUsageForRun(runStartedAt) {
+    if (sseUsageCache.size === 0)
+        return null;
+    const entries = Array.from(sseUsageCache.entries()).sort((a, b) => a[0] - b[0]);
+    let selectedKey = null;
+    if (typeof runStartedAt === 'number') {
+        const toleranceMs = 1000;
+        for (const [key, usage] of entries) {
+            if (usage.timestamp + toleranceMs >= runStartedAt) {
+                selectedKey = key;
+                break;
+            }
+        }
+    }
+    if (selectedKey === null)
+        selectedKey = entries[0][0];
+    const usage = sseUsageCache.get(selectedKey) ?? null;
+    sseUsageCache.delete(selectedKey);
     // Also trim old cache entries
     if (sseUsageCache.size > 100) {
         const keys = Array.from(sseUsageCache.keys()).sort((a, b) => a - b);
@@ -190,14 +478,61 @@ function popLatestSseUsage() {
     return usage;
 }
 const runInputs = new Map();
+/** Track latest completed tool-call timestamp for each run (for thinking timeline alignment) */
+const runLastToolCompletedAt = new Map();
+/** Track latest assistant stream event timestamp for each run (for thinking end-time clamping) */
+const runLastAssistantStreamAt = new Map();
+/** Track first assistant stream event timestamp for each run (assistant start-time anchor) */
+const runFirstAssistantStreamAt = new Map();
+function summarizePromptMessages(messages) {
+    const roleCounts = {};
+    const firstRoles = [];
+    if (!Array.isArray(messages) || messages.length === 0) {
+        return { messageCount: 0, roleCounts, firstRoles };
+    }
+    for (let i = 0; i < messages.length; i++) {
+        const item = messages[i];
+        if (!item || typeof item !== 'object')
+            continue;
+        const role = String(item.role || 'unknown');
+        roleCounts[role] = (roleCounts[role] ?? 0) + 1;
+        if (firstRoles.length < 20) {
+            firstRoles.push(role);
+        }
+    }
+    return {
+        messageCount: messages.length,
+        roleCounts,
+        firstRoles,
+    };
+}
+function makeToolTimingKey(sessionId, runId, toolName, toolCallId) {
+    if (toolCallId)
+        return `tc:id:${toolCallId}`;
+    return `tc:fallback:${sessionId}:${runId ?? 'norun'}:${toolName}`;
+}
 /**
  * Periodically clean up stale Map entries to prevent memory leaks when llm_input fires but llm_output does not
  */
 function cleanupStaleMaps() {
     const now = Date.now();
-    for (const [key, startTime] of runStartTimes) {
+    for (const [key, startTime] of llmRunStartTimes) {
         if (now - startTime > MAP_ENTRY_TTL_MS) {
-            runStartTimes.delete(key);
+            llmRunStartTimes.delete(key);
+            runSessionIds.delete(key);
+            runInputs.delete(key);
+            runLastToolCompletedAt.delete(key);
+            runLastAssistantStreamAt.delete(key);
+            runFirstAssistantStreamAt.delete(key);
+        }
+    }
+    for (const [key, queue] of toolCallStartTimes) {
+        const filtered = queue.filter((t) => now - t <= MAP_ENTRY_TTL_MS);
+        if (filtered.length === 0) {
+            toolCallStartTimes.delete(key);
+        }
+        else if (filtered.length !== queue.length) {
+            toolCallStartTimes.set(key, filtered);
         }
     }
     // runInputs has no timestamps; trim oldest entries when exceeding size limit
@@ -230,6 +565,18 @@ function cleanupStaleMaps() {
             sessionStatsMap.delete(entries[i][0]);
         }
     }
+    for (const [key, pending] of pendingRuntimeEvents) {
+        if (now - pending.lastEventAt > MAP_ENTRY_TTL_MS) {
+            if (pending.timer)
+                clearTimeout(pending.timer);
+            pendingRuntimeEvents.delete(key);
+        }
+    }
+    for (const [key, ts] of replayedAssistantMessages) {
+        if (now - ts > MAP_ENTRY_TTL_MS) {
+            replayedAssistantMessages.delete(key);
+        }
+    }
 }
 /**
  * Extract image/media metadata from historyMessages
@@ -446,6 +793,7 @@ let _alertBuffer = [];
 let _alertFlushTimer = null;
 let _mapCleanupTimer = null;
 let _checkpointTimer = null;
+let _unsubscribeAgentEvents = null;
 function activate(api) {
     const rawConfig = (api.pluginConfig || api.config || {});
     const config = (0, config_1.resolveConfig)(rawConfig);
@@ -482,9 +830,38 @@ function activate(api) {
         console.log('[openclaw-observability] Reusing existing writer/buffer (skipping duplicate init)');
     }
     const redactor = new redaction_1.Redactor(config.redaction);
-    // Security scanner
-    const securityConfig = (0, scanner_1.resolveSecurityConfig)(config.security);
+    const stateDir = process.env.OPENCLAW_STATE_DIR ?? path.join(os.homedir(), '.openclaw');
+    // Security scanner (supports runtime rule toggles from UI)
+    const securityConfigPath = path.join(stateDir, 'data', 'observability-security-config.json');
+    let securityConfig = (0, scanner_1.resolveSecurityConfig)(config.security);
+    try {
+        if (fs.existsSync(securityConfigPath)) {
+            const raw = fs.readFileSync(securityConfigPath, 'utf8');
+            const parsed = JSON.parse(raw);
+            securityConfig = (0, scanner_1.resolveSecurityConfig)({
+                ...securityConfig,
+                ...parsed,
+                rules: {
+                    ...securityConfig.rules,
+                    ...(parsed.rules ?? {}),
+                },
+            });
+            console.log('[openclaw-observability-security] Loaded runtime security config from ' + securityConfigPath);
+        }
+    }
+    catch (e) {
+        console.warn('[openclaw-observability-security] Failed to load runtime security config:', e.message);
+    }
     const securityScanner = new scanner_1.SecurityScanner(securityConfig);
+    function persistSecurityConfig(nextConfig) {
+        try {
+            fs.mkdirSync(path.dirname(securityConfigPath), { recursive: true });
+            fs.writeFileSync(securityConfigPath, JSON.stringify(nextConfig, null, 2), 'utf8');
+        }
+        catch (e) {
+            console.error('[openclaw-observability-security] Failed to persist runtime security config:', e);
+        }
+    }
     function startServices() {
         if (_servicesStarted)
             return;
@@ -530,15 +907,49 @@ function activate(api) {
                 'Please configure via Dashboard (Settings → Plugins → openclaw-observability Config) then restart.');
         }
     }
-    // Observability UI auth: prefer plugin ui.accessToken, else gateway token from config, else built-in default
-    const gatewayAuth = api.config?.gateway?.auth;
-    const gatewayToken = (typeof gatewayAuth?.token === 'string' && gatewayAuth.token.length > 0)
-        ? gatewayAuth.token
-        : process.env.OPENCLAW_GATEWAY_TOKEN;
-    const observabilityToken = config.ui?.accessToken ?? gatewayToken ?? 'openclaw-observability-default';
+    // Observability UI auth: read gateway.auth.token from same config file as gateway; if missing or empty, no auth (allow all)
+    const openclawConfigPath = process.env.OPENCLAW_CONFIG_PATH ?? path.join(stateDir, 'openclaw.json');
+    let observabilityToken;
+    try {
+        if (fs.existsSync(openclawConfigPath)) {
+            const raw = fs.readFileSync(openclawConfigPath, 'utf8');
+            let t;
+            try {
+                const parsed = JSON.parse(raw);
+                t = parsed?.gateway?.auth?.token;
+            }
+            catch {
+                // Config may be JSON5 (comments, unescaped newlines in strings); extract token with regex
+                const m = raw.match(/"gateway"\s*:\s*\{[\s\S]*?"auth"\s*:\s*\{[\s\S]*?"token"\s*:\s*"((?:[^"\\]|\\.)*)"/);
+                t = m?.[1]?.replace(/\\(.)/g, '$1');
+            }
+            if (typeof t === 'string' && t.length > 0)
+                observabilityToken = t;
+        }
+    }
+    catch (e) {
+        console.warn('[openclaw-observability] Failed to read gateway token from config:', e.message);
+    }
+    if (observabilityToken) {
+        console.log('[openclaw-observability] Observability UI auth: token required (from ' + openclawConfigPath + ')');
+    }
+    else {
+        console.log('[openclaw-observability] Observability UI auth: no token, allow all');
+    }
     // Always register HTTP routes on the current api (the latest api serves HTTP)
     if (typeof api.registerHttpRoute === 'function') {
-        (0, routes_1.registerAuditRoutes)(api.registerHttpRoute.bind(api), writer, observabilityToken);
+        (0, routes_1.registerAuditRoutes)(api.registerHttpRoute.bind(api), writer, observabilityToken, {
+            getConfig: () => securityScanner.getConfig(),
+            updateConfig: (patch) => {
+                const next = securityScanner.updateConfig(patch);
+                persistSecurityConfig(next);
+                console.log(`[openclaw-observability-security] Runtime config updated: enabled=${next.enabled} ` +
+                    `secretLeakage=${next.rules.secretLeakage} highRiskOps=${next.rules.highRiskOps} dataExfiltration=${next.rules.dataExfiltration} ` +
+                    `promptInjection=${next.rules.promptInjection} customRegex=${next.rules.customRegex} chainDetection=${next.rules.chainDetection} ` +
+                    `customRegexRules=${next.customRegexRules.length}`);
+                return next;
+            },
+        });
     }
     else {
         console.warn('[openclaw-observability] registerHttpRoute not available — observability UI disabled');
@@ -550,30 +961,28 @@ function activate(api) {
         const session = updateSessionStats(action.sessionId, action.modelName, action.userId, tokens);
         void buffer.addSession(session);
         // --- Security scan ---
-        if (securityConfig.enabled) {
-            try {
-                const alerts = securityScanner.scan(action);
-                if (alerts.length > 0) {
-                    // Prevent _alertBuffer from growing unbounded
-                    if (_alertBuffer.length + alerts.length > MAX_ALERT_BUFFER) {
-                        const overflow = _alertBuffer.length + alerts.length - MAX_ALERT_BUFFER;
-                        _alertBuffer.splice(0, overflow);
-                        console.warn(`[openclaw-observability-security] Alert buffer overflow, dropped ${overflow} oldest alerts`);
-                    }
-                    _alertBuffer.push(...alerts);
-                    for (const alert of alerts) {
-                        const icon = alert.severity === 'critical' ? '🔴' : alert.severity === 'warn' ? '🟡' : 'ℹ️';
-                        console.log(`[openclaw-observability-security] ${icon} ${alert.severity.toUpperCase()} ${alert.ruleId}: ${alert.ruleName} — ${alert.finding} (session=${alert.sessionId})`);
-                    }
-                    // Flush immediately on CRITICAL alerts
-                    if (alerts.some(a => a.severity === 'critical')) {
-                        void flushAlerts();
-                    }
+        try {
+            const alerts = securityScanner.scan(action);
+            if (alerts.length > 0) {
+                // Prevent _alertBuffer from growing unbounded
+                if (_alertBuffer.length + alerts.length > MAX_ALERT_BUFFER) {
+                    const overflow = _alertBuffer.length + alerts.length - MAX_ALERT_BUFFER;
+                    _alertBuffer.splice(0, overflow);
+                    console.warn(`[openclaw-observability-security] Alert buffer overflow, dropped ${overflow} oldest alerts`);
+                }
+                _alertBuffer.push(...alerts);
+                for (const alert of alerts) {
+                    const icon = alert.severity === 'critical' ? '🔴' : alert.severity === 'warn' ? '🟡' : 'ℹ️';
+                    console.log(`[openclaw-observability-security] ${icon} ${alert.severity.toUpperCase()} ${alert.ruleId}: ${alert.ruleName} — ${alert.finding} (session=${alert.sessionId})`);
+                }
+                // Flush immediately on CRITICAL alerts
+                if (alerts.some(a => a.severity === 'critical')) {
+                    void flushAlerts();
                 }
             }
-            catch (err) {
-                console.error('[openclaw-observability-security] Scan error:', err);
-            }
+        }
+        catch (err) {
+            console.error('[openclaw-observability-security] Scan error:', err);
         }
     }
     /** Batch-write security alerts */
@@ -590,12 +999,6 @@ function activate(api) {
             _alertBuffer.unshift(...batch);
         }
     }
-    /** Safely truncate a string */
-    function truncate(s, max = 500) {
-        if (typeof s !== 'string')
-            return '';
-        return s.length > max ? s.substring(0, max) + '…' : s;
-    }
     /** Convert bytes to human-readable format */
     function formatBytes(bytes) {
         if (bytes < 1024)
@@ -604,6 +1007,141 @@ function activate(api) {
             return `${(bytes / 1024).toFixed(1)} KB`;
         return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
     }
+    function flushPendingRuntimeEvent(key) {
+        const pending = pendingRuntimeEvents.get(key);
+        if (!pending)
+            return;
+        if (pending.timer) {
+            clearTimeout(pending.timer);
+            pending.timer = null;
+        }
+        pendingRuntimeEvents.delete(key);
+        const text = pending.text.trim();
+        if (!text)
+            return;
+        recordAction(makeAction(pending.sessionId, {
+            actionType: pending.actionType,
+            actionName: `${pending.stream}_stream`,
+            inputParams: redactor.redact({
+                runId: pending.runId,
+                stream: pending.stream,
+            }),
+            outputResult: redactor.redact({
+                text,
+                length: text.length,
+            }),
+            durationMs: Math.max(0, pending.lastEventAt - pending.startedAt),
+            createdAt: new Date(pending.lastEventAt),
+        }));
+    }
+    function queueRuntimeTextEvent(params) {
+        const key = `${params.runId}:${params.stream}`;
+        const existing = pendingRuntimeEvents.get(key);
+        if (existing) {
+            existing.lastEventAt = params.ts;
+            existing.text = params.text;
+            if (existing.timer)
+                clearTimeout(existing.timer);
+            existing.timer = setTimeout(() => flushPendingRuntimeEvent(key), 900);
+            if (existing.timer && typeof existing.timer === 'object' && 'unref' in existing.timer) {
+                existing.timer.unref();
+            }
+            return;
+        }
+        const timer = setTimeout(() => flushPendingRuntimeEvent(key), 900);
+        if (timer && typeof timer === 'object' && 'unref' in timer) {
+            timer.unref();
+        }
+        pendingRuntimeEvents.set(key, {
+            actionType: params.actionType,
+            sessionId: params.sessionId,
+            runId: params.runId,
+            stream: params.stream,
+            startedAt: params.ts,
+            lastEventAt: params.ts,
+            text: params.text,
+            timer,
+        });
+    }
+    function flushPendingRuntimeEventsForRun(runId) {
+        const keys = Array.from(pendingRuntimeEvents.keys()).filter((key) => key.startsWith(`${runId}:`));
+        keys.forEach((key) => flushPendingRuntimeEvent(key));
+    }
+    if (!_unsubscribeAgentEvents && typeof api.runtime?.events?.onAgentEvent === 'function') {
+        const unsubscribe = api.runtime.events.onAgentEvent((evt) => {
+            try {
+                if (!evt?.runId || !evt?.stream)
+                    return;
+                const sessionId = runSessionIds.get(evt.runId);
+                if (!sessionId || sessionId === 'unknown')
+                    return;
+                if (evt.stream === 'assistant' || evt.stream === 'thinking') {
+                    const fullText = typeof evt.data?.text === 'string'
+                        ? evt.data.text
+                        : typeof evt.data?.delta === 'string'
+                            ? evt.data.delta
+                            : '';
+                    if (!fullText.trim())
+                        return;
+                    runtimeTextSeen.add(getRuntimeTextSeenKey(evt.runId, evt.stream));
+                    if (evt.stream === 'assistant') {
+                        if (!runFirstAssistantStreamAt.has(evt.runId)) {
+                            runFirstAssistantStreamAt.set(evt.runId, typeof evt.ts === 'number' ? evt.ts : Date.now());
+                        }
+                        runLastAssistantStreamAt.set(evt.runId, typeof evt.ts === 'number' ? evt.ts : Date.now());
+                    }
+                    queueRuntimeTextEvent({
+                        sessionId,
+                        runId: evt.runId,
+                        stream: evt.stream,
+                        actionType: evt.stream === 'assistant' ? types_1.ActionType.AssistantStream : types_1.ActionType.Thinking,
+                        ts: typeof evt.ts === 'number' ? evt.ts : Date.now(),
+                        text: fullText,
+                    });
+                    return;
+                }
+                if (evt.stream === 'tool') {
+                    flushPendingRuntimeEventsForRun(evt.runId);
+                    const phase = typeof evt.data?.phase === 'string' ? evt.data.phase : '';
+                    const name = typeof evt.data?.name === 'string' ? evt.data.name : 'unknown';
+                    if (phase !== 'start' && phase !== 'update')
+                        return;
+                    const payload = { runId: evt.runId, phase };
+                    if (typeof evt.data?.toolCallId === 'string')
+                        payload.toolCallId = evt.data.toolCallId;
+                    if (phase === 'update' && evt.data?.partialResult !== undefined) {
+                        payload.partialResult = evt.data.partialResult;
+                    }
+                    recordAction(makeAction(sessionId, {
+                        actionType: types_1.ActionType.ToolUpdate,
+                        actionName: phase === 'start' ? `tool_start:${name}` : `tool_update:${name}`,
+                        inputParams: redactor.redact(payload),
+                        createdAt: new Date(typeof evt.ts === 'number' ? evt.ts : Date.now()),
+                    }));
+                    return;
+                }
+                if (evt.stream === 'lifecycle') {
+                    const phase = typeof evt.data?.phase === 'string' ? evt.data.phase : '';
+                    if (phase === 'end' || phase === 'error') {
+                        flushPendingRuntimeEventsForRun(evt.runId);
+                    }
+                }
+            }
+            catch (err) {
+                console.error('[openclaw-observability] Error handling runtime agent event:', err);
+            }
+        });
+        if (typeof unsubscribe === 'function') {
+            _unsubscribeAgentEvents = unsubscribe;
+        }
+        console.log('[openclaw-observability] Runtime agent event listener attached');
+    }
+    else if (_unsubscribeAgentEvents) {
+        console.log('[openclaw-observability] Reusing existing runtime agent event listener');
+    }
+    else {
+        console.log('[openclaw-observability] Runtime agent events unavailable on this OpenClaw build');
+    }
     // =====================================================================
     //  1. Agent lifecycle
     // =====================================================================
@@ -620,7 +1158,7 @@ function activate(api) {
                 actionType: types_1.ActionType.ModelResolve,
                 actionName: 'before_model_resolve',
                 userId: c?.agentId,
-                inputParams: redactor.redact({ prompt: truncate(e.prompt) }),
+                inputParams: redactor.redact({ prompt: e.prompt }),
             }));
             console.log(`[openclaw-observability] before_model_resolve: session=${sid}`);
         }
@@ -634,13 +1172,16 @@ function activate(api) {
             const e = event;
             const c = ctx;
             const sid = resolveSessionId(undefined, c?.sessionId);
+            const summary = summarizePromptMessages(e.messages);
             recordAction(makeAction(sid, {
                 actionType: types_1.ActionType.PromptBuild,
                 actionName: 'before_prompt_build',
                 userId: c?.agentId,
                 inputParams: redactor.redact({
-                    prompt: truncate(e.prompt),
-                    messageCount: Array.isArray(e.messages) ? e.messages.length : 0,
+                    prompt: e.prompt,
+                    messageCount: summary.messageCount,
+                    messageRoleCounts: summary.roleCounts,
+                    firstMessageRoles: summary.firstRoles,
                 }),
             }));
             console.log(`[openclaw-observability] before_prompt_build: session=${sid} msgs=${Array.isArray(e.messages) ? e.messages.length : '?'}`);
@@ -686,10 +1227,46 @@ function activate(api) {
             const sctx = getSessionCtx(sid);
             // Parse image/media metadata from historyMessages
             const media = extractMediaMeta(e.historyMessages ?? []);
+            const modelName = `${e.provider}/${e.model}`;
+            const priorAssistant = extractToolUseAssistantMessage(e.historyMessages ?? []);
+            if (priorAssistant && e.runId) {
+                const replayKey = [
+                    sid,
+                    e.runId,
+                    priorAssistant.messageId ?? String(priorAssistant.timestampMs ?? 'notime'),
+                    priorAssistant.toolCallIds.join(','),
+                ].join(':');
+                if (!replayedAssistantMessages.has(replayKey)) {
+                    replayedAssistantMessages.set(replayKey, Date.now());
+                    const replayAt = new Date(priorAssistant.timestampMs ?? Math.max(Date.now() - 5, 0));
+                    const replayMeta = {
+                        runId: e.runId,
+                        source: 'history_message',
+                        synthetic: true,
+                        stopReason: priorAssistant.stopReason ?? 'toolUse',
+                        messageId: priorAssistant.messageId,
+                        toolCallIds: priorAssistant.toolCallIds,
+                    };
+                    if (priorAssistant.texts.length > 0) {
+                        const assistantText = priorAssistant.texts.join('\n\n');
+                        recordAction(makeAction(sid, {
+                            actionType: types_1.ActionType.AssistantStream,
+                            actionName: 'assistant_stream',
+                            modelName,
+                            inputParams: redactor.redact(replayMeta),
+                            outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
+                            createdAt: new Date(replayAt.getTime() + 1),
+                            userId: c?.agentId,
+                        }));
+                    }
+                }
+            }
             if (e.runId) {
-                runStartTimes.set(e.runId, Date.now());
+                llmRunStartTimes.set(e.runId, Date.now());
+                runSessionIds.set(e.runId, sid);
                 runInputs.set(e.runId, {
                     prompt: e.prompt,
+                    systemPrompt: typeof e.systemPrompt === 'string' ? e.systemPrompt : undefined,
                     imagesCount: e.imagesCount,
                     media: media.length > 0 ? media : undefined,
                 });
@@ -701,7 +1278,7 @@ function activate(api) {
             saveChannelToContext(sid, identifiedChannel);
             // Use identified channel or existing context channel
             const channelId = identifiedChannel !== 'webchat' ? identifiedChannel : (sctx.channelId || 'webchat');
-            sctx.modelName = `${e.provider}/${e.model}`;
+            sctx.modelName = modelName;
             console.log(`[openclaw-observability] llm_input: session=${sid} model=${e.provider}/${e.model} runId=${e.runId} channel=${channelId} images=${e.imagesCount ?? 0} mediaParts=${media.length}`);
         }
         catch (err) {
@@ -715,9 +1292,12 @@ function activate(api) {
             const c = ctx;
             // Calculate duration
             let durationMs = null;
-            if (e.runId && runStartTimes.has(e.runId)) {
-                durationMs = Date.now() - runStartTimes.get(e.runId);
-                runStartTimes.delete(e.runId);
+            const runStartedAt = e.runId ? llmRunStartTimes.get(e.runId) : undefined;
+            const runLastToolAt = e.runId ? runLastToolCompletedAt.get(e.runId) : undefined;
+            const runAssistantAt = e.runId ? runLastAssistantStreamAt.get(e.runId) : undefined;
+            const runAssistantFirstAt = e.runId ? runFirstAssistantStreamAt.get(e.runId) : undefined;
+            if (typeof runStartedAt === 'number') {
+                durationMs = Date.now() - runStartedAt;
             }
             // Retrieve cached user input
             const cachedInput = e.runId ? runInputs.get(e.runId) : undefined;
@@ -748,7 +1328,7 @@ function activate(api) {
             }
             // Strategy 3: SSE usage captured by fetch interceptor (stream_options injection)
             if (promptTokens === null && completionTokens === null) {
-                const sseUsage = popLatestSseUsage();
+                const sseUsage = consumeSseUsageForRun(runStartedAt);
                 if (sseUsage) {
                     promptTokens = sseUsage.input || null;
                     completionTokens = sseUsage.output || null;
@@ -758,11 +1338,22 @@ function activate(api) {
             }
             const totalTokens = (promptTokens ?? 0) + (completionTokens ?? 0);
             const modelName = `${e.provider}/${e.model}`;
+            const fallbackArtifacts = extractAssistantArtifacts(e.lastAssistant);
+            const assistantTexts = Array.isArray(e.assistantTexts) && e.assistantTexts.some((text) => typeof text === 'string' && text.trim())
+                ? e.assistantTexts.filter((text) => typeof text === 'string' && text.trim().length > 0)
+                : fallbackArtifacts.texts;
             // Build inputParams with media metadata
             const inputData = {
+                runId: e.runId,
                 userMessage: cachedInput?.prompt ?? '',
                 imagesCount: cachedInput?.imagesCount ?? 0,
             };
+            if (typeof cachedInput?.systemPrompt === 'string' && cachedInput.systemPrompt.length > 0) {
+                inputData.systemPrompt = cachedInput.systemPrompt;
+                inputData.systemPromptHash = (0, node_crypto_1.createHash)('sha256')
+                    .update(cachedInput.systemPrompt, 'utf8')
+                    .digest('hex');
+            }
             if (cachedInput?.media && cachedInput.media.length > 0) {
                 inputData.media = cachedInput.media.map((m) => ({
                     mimeType: m.mimeType,
@@ -782,14 +1373,127 @@ function activate(api) {
                 modelName,
                 inputParams: redactor.redact(inputData),
                 outputResult: redactor.redact({
-                    assistantTexts: e.assistantTexts ?? [],
+                    runId: e.runId,
+                    assistantTexts,
+                    thinking: fallbackArtifacts.thinking,
+                    usage: {
+                        promptTokens,
+                        completionTokens,
+                        totalTokens,
+                        cacheRead,
+                        cacheWrite,
+                    },
                 }),
                 promptTokens,
                 completionTokens,
                 durationMs,
                 userId: c?.agentId,
             }), totalTokens);
+            const endedAt = Date.now();
+            const syntheticTs = endedAt;
+            if (e.runId) {
+                const transcriptToolUseMessages = extractToolUseAssistantMessagesFromTranscript({
+                    sessionId: sid,
+                    runStartedAt,
+                    runEndedAt: endedAt,
+                });
+                for (const msg of transcriptToolUseMessages) {
+                    const replayKey = `${sid}:${e.runId}:${msg.messageId}`;
+                    if (replayedAssistantMessages.has(replayKey))
+                        continue;
+                    replayedAssistantMessages.set(replayKey, Date.now());
+                    const replayMeta = {
+                        runId: e.runId,
+                        source: 'transcript_message',
+                        synthetic: true,
+                        stopReason: msg.stopReason ?? 'toolUse',
+                        messageId: msg.messageId,
+                        toolCallIds: msg.toolCallIds,
+                    };
+                    const replayAt = new Date(msg.timestampMs);
+                    if (msg.thinking.length > 0) {
+                        const thinkingText = msg.thinking.join('\n\n');
+                        recordAction(makeAction(sid, {
+                            actionType: types_1.ActionType.Thinking,
+                            actionName: 'thinking',
+                            modelName,
+                            inputParams: redactor.redact(replayMeta),
+                            outputResult: redactor.redact({ text: thinkingText, length: thinkingText.length }),
+                            createdAt: replayAt,
+                            userId: c?.agentId,
+                        }));
+                    }
+                    if (msg.texts.length > 0) {
+                        const assistantText = msg.texts.join('\n\n');
+                        recordAction(makeAction(sid, {
+                            actionType: types_1.ActionType.AssistantStream,
+                            actionName: 'assistant_stream',
+                            modelName,
+                            inputParams: redactor.redact(replayMeta),
+                            outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
+                            createdAt: new Date(msg.timestampMs + 1),
+                            userId: c?.agentId,
+                        }));
+                    }
+                }
+                if (assistantTexts.length > 0 &&
+                    !runtimeTextSeen.has(getRuntimeTextSeenKey(e.runId, 'assistant'))) {
+                    var assistantText = assistantTexts.join('\n\n');
+                    recordAction(makeAction(sid, {
+                        actionType: types_1.ActionType.AssistantStream,
+                        actionName: 'assistant_stream',
+                        modelName,
+                        inputParams: redactor.redact({ runId: e.runId, stream: 'assistant', synthetic: true }),
+                        outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
+                        createdAt: new Date(syntheticTs - 1),
+                        userId: c?.agentId,
+                    }));
+                }
+                if (fallbackArtifacts.thinking.length > 0 &&
+                    !runtimeTextSeen.has(getRuntimeTextSeenKey(e.runId, 'thinking'))) {
+                    const thinkingText = fallbackArtifacts.thinking.join('\n\n');
+                    let thinkingEndAt = Math.max(syntheticTs - 2, 0);
+                    // Anchor thinking end at assistant start time when available.
+                    // Priority: pending assistant stream start -> first observed assistant stream ts.
+                    const pendingAssistant = e.runId
+                        ? pendingRuntimeEvents.get(`${e.runId}:assistant`)
+                        : undefined;
+                    const assistantStartAt = (pendingAssistant && typeof pendingAssistant.startedAt === 'number')
+                        ? pendingAssistant.startedAt
+                        : (typeof runAssistantFirstAt === 'number' ? runAssistantFirstAt : undefined);
+                    if (typeof assistantStartAt === 'number') {
+                        thinkingEndAt = Math.min(thinkingEndAt, Math.max(assistantStartAt, 0));
+                    }
+                    else if (typeof runAssistantAt === 'number') {
+                        thinkingEndAt = Math.min(thinkingEndAt, Math.max(runAssistantAt - 1, 0));
+                    }
+                    let inferredThinkingDuration = null;
+                    if (typeof runLastToolAt === 'number' && runLastToolAt < thinkingEndAt) {
+                        inferredThinkingDuration = Math.max(1, thinkingEndAt - runLastToolAt);
+                    }
+                    recordAction(makeAction(sid, {
+                        actionType: types_1.ActionType.Thinking,
+                        actionName: 'thinking',
+                        modelName,
+                        inputParams: redactor.redact({ runId: e.runId, stream: 'thinking', synthetic: true }),
+                        outputResult: redactor.redact({ text: thinkingText, length: thinkingText.length }),
+                        createdAt: new Date(thinkingEndAt),
+                        durationMs: inferredThinkingDuration,
+                        userId: c?.agentId,
+                    }));
+                }
+            }
             console.log(`[openclaw-observability] llm_output: session=${e.sessionId} model=${e.model} duration=${durationMs}ms tokens=${promptTokens ?? '?'}/${completionTokens ?? '?'} cache_r=${cacheRead ?? '-'} cache_w=${cacheWrite ?? '-'}`);
+            if (e.runId) {
+                flushPendingRuntimeEventsForRun(e.runId);
+                llmRunStartTimes.delete(e.runId);
+                runSessionIds.delete(e.runId);
+                runLastToolCompletedAt.delete(e.runId);
+                runLastAssistantStreamAt.delete(e.runId);
+                runFirstAssistantStreamAt.delete(e.runId);
+                runtimeTextSeen.delete(getRuntimeTextSeenKey(e.runId, 'assistant'));
+                runtimeTextSeen.delete(getRuntimeTextSeenKey(e.runId, 'thinking'));
+            }
         }
         catch (err) {
             console.error('[openclaw-observability] Error in llm_output:', err);
@@ -913,8 +1617,9 @@ function activate(api) {
                 modelName: '',
                 inputParams: redactor.redact({
                     to: e.to,
-                    content: truncate(e.content, 1000),
+                    content: e.content,
                     channelId: c?.channelId,
+                    metadata: e.metadata,
                 }),
             }));
             console.log(`[openclaw-observability] message_sending: to=${e.to} channel=${c?.channelId}`);
@@ -958,13 +1663,14 @@ function activate(api) {
         try {
             const e = event;
             const c = ctx;
-            // Use toolCallId as timing key to avoid collision with llm_input runId
-            // Note: cannot use Date.now() as fallback, otherwise before and after generate inconsistent keys
+            // Use a stable key; when toolCallId is missing, use session/run/tool fallback key.
             const callId = e.toolCallId || c?.toolCallId || '';
-            const timingKey = callId ? `tc:${callId}` : `tc:${e.toolName}`;
-            runStartTimes.set(timingKey, Date.now());
-            // Update per-session context
             const sid = resolveSessionId(undefined, c?.sessionId);
+            const timingKey = makeToolTimingKey(sid, c?.runId || e.runId, e.toolName, callId || undefined);
+            const queue = toolCallStartTimes.get(timingKey) ?? [];
+            queue.push(Date.now());
+            toolCallStartTimes.set(timingKey, queue);
+            // Update per-session context
             const sctx = getSessionCtx(sid);
             if (c?.agentId)
                 sctx.userId = c.agentId;
@@ -984,14 +1690,29 @@ function activate(api) {
             const sessionId = resolveSessionId(undefined, c?.sessionId);
             // Calculate duration — use toolCallId as key to not interfere with LLM runId timing
             const callId = e.toolCallId || c?.toolCallId || '';
-            const timingKey = callId ? `tc:${callId}` : `tc:${e.toolName}`;
+            const timingKey = makeToolTimingKey(sessionId, c?.runId || e.runId, e.toolName, callId || undefined);
             let durationMs = e.durationMs ?? null;
-            if (durationMs === null && runStartTimes.has(timingKey)) {
-                durationMs = Date.now() - runStartTimes.get(timingKey);
+            const queue = toolCallStartTimes.get(timingKey);
+            const startAt = queue && queue.length > 0 ? queue.shift() : undefined;
+            if (queue && queue.length > 0) {
+                toolCallStartTimes.set(timingKey, queue);
+            }
+            else {
+                toolCallStartTimes.delete(timingKey);
+            }
+            if (durationMs === null && typeof startAt === 'number') {
+                durationMs = Date.now() - startAt;
+            }
+            if (typeof (e.runId || c?.runId) === 'string') {
+                runLastToolCompletedAt.set(String(e.runId || c?.runId), Date.now());
             }
-            runStartTimes.delete(timingKey);
             // BUG FIX: use correct field name params (not parameters/input/args)
-            const inputParams = e.params ? redactor.redact(e.params) : null;
+            const rawInputParams = e.params ? { ...e.params } : {};
+            if (callId)
+                rawInputParams.toolCallId = callId;
+            if (e.runId || c?.runId)
+                rawInputParams.runId = e.runId || c?.runId;
+            const inputParams = redactor.redact(rawInputParams);
             // Detect image content in tool results
             let outputData = null;
             if (e.error) {
@@ -1055,29 +1776,6 @@ function activate(api) {
             console.error('[openclaw-observability] Error in after_tool_call:', err);
         }
     });
-    // tool_result_persist: tool result persistence (sync hook)
-    api.on('tool_result_persist', (event, ctx) => {
-        try {
-            const e = event;
-            const c = ctx;
-            const name = e.toolName || c?.toolName || 'unknown';
-            const sid = lastFallbackSessionId;
-            if (sid === 'unknown')
-                return;
-            recordAction(makeAction(sid, {
-                actionType: types_1.ActionType.ToolPersist,
-                actionName: `tool_persist:${name}`,
-                inputParams: redactor.redact({
-                    toolName: name,
-                    toolCallId: e.toolCallId,
-                    isSynthetic: e.isSynthetic,
-                }),
-            }));
-        }
-        catch (err) {
-            console.error('[openclaw-observability] Error in tool_result_persist:', err);
-        }
-    });
     // =====================================================================
     //  6. Session lifecycle
     // =====================================================================
@@ -1181,6 +1879,8 @@ function activate(api) {
                     reason: e.reason,
                     outcome: e.outcome,
                     error: e.error,
+                    runId: e.runId,
+                    endedAt: e.endedAt,
                 }),
             }));
             console.log(`[openclaw-observability] subagent_ended: target=${e.targetSessionKey} outcome=${e.outcome}`);
@@ -1240,6 +1940,11 @@ function activate(api) {
     return {
         deactivate: async () => {
             console.log('[openclaw-observability] Deactivating plugin...');
+            if (_unsubscribeAgentEvents) {
+                _unsubscribeAgentEvents();
+                _unsubscribeAgentEvents = null;
+            }
+            Array.from(pendingRuntimeEvents.keys()).forEach((key) => flushPendingRuntimeEvent(key));
             // Stop module-level timers
             if (_alertFlushTimer) {
                 clearInterval(_alertFlushTimer);
@@ -1268,10 +1973,12 @@ function activate(api) {
             _alertBuffer = [];
             _servicesStarted = false;
             // Reset function-level state
-            runStartTimes.clear();
+            llmRunStartTimes.clear();
+            runSessionIds.clear();
+            toolCallStartTimes.clear();
             runInputs.clear();
+            pendingRuntimeEvents.clear();
             sseUsageCache.clear();
-            latestSseUsage = null;
             sessionStatsMap.clear();
             sessionContextMap.clear();
             securityScanner.reset();