openclaw-observability 2026.4.1 → 2026.4.21

package/dist/index.js

@@ -53,6 +53,7 @@ const types_1 = require("./types");
 const redaction_1 = require("./redaction");
 const mysql_writer_1 = require("./storage/mysql-writer");
 const duckdb_local_writer_1 = require("./storage/duckdb-local-writer");
+const cloud_export_writer_1 = require("./storage/cloud-export-writer");
 const buffer_1 = require("./storage/buffer");
 const routes_1 = require("./web/routes");
 const scanner_1 = require("./security/scanner");
@@ -62,6 +63,7 @@ const os = __importStar(require("node:os"));
 const node_crypto_1 = require("node:crypto");
 const runtime_1 = require("./runtime");
 const register_observability_gateway_1 = require("./gateway/register-observability-gateway");
+const replay_runtime_1 = require("./llm/replay-runtime");
 const hooks_1 = require("./hooks");
 /* ------------------------------------------------------------------ */
 /*  Runtime state                                                       */
@@ -82,6 +84,100 @@ const pendingRuntimeEvents = new Map();
 const runtimeTextSeen = new Set();
 const replayedAssistantMessages = new Map();
 const sessionTranscriptPathCache = new Map();
+const runParentLinks = new Map();
+function normalizeOtlpBasePath(raw) {
+    const trimmed = String(raw || '').trim();
+    if (!trimmed)
+        return '/plugins/observability/api/otel';
+    const withLeading = trimmed.startsWith('/') ? trimmed : `/${trimmed}`;
+    return withLeading.replace(/\/+$/, '') || '/plugins/observability/api/otel';
+}
+function resolveOpenclawStateDir() {
+    const envDir = String(process.env.OPENCLAW_HOME || process.env.OPENCLAW_STATE_DIR || '').trim();
+    if (envDir)
+        return path.resolve(envDir);
+    // Runtime layout: <stateDir>/extensions/openclaw-observability/dist/index.js
+    const fromModule = path.resolve(__dirname, '..', '..', '..');
+    if (path.basename(fromModule).startsWith('.openclaw'))
+        return fromModule;
+    return path.join(os.homedir(), '.openclaw');
+}
+function ensureDiagnosticsOtelBootstrap(params) {
+    const { configPath, otlpPath, token } = params;
+    try {
+        if (!fs.existsSync(configPath))
+            return;
+        const raw = fs.readFileSync(configPath, 'utf8');
+        const parsed = JSON.parse(raw);
+        const gateway = (parsed.gateway && typeof parsed.gateway === 'object') ? parsed.gateway : {};
+        const gatewayPort = Number(gateway.port);
+        const port = Number.isFinite(gatewayPort) && gatewayPort > 0 ? Math.floor(gatewayPort) : 18789;
+        const endpoint = `http://127.0.0.1:${port}${normalizeOtlpBasePath(otlpPath)}`;
+        const diagnostics = (parsed.diagnostics && typeof parsed.diagnostics === 'object') ? parsed.diagnostics : {};
+        const currentOtel = (diagnostics.otel && typeof diagnostics.otel === 'object') ? diagnostics.otel : {};
+        const headers = (currentOtel.headers && typeof currentOtel.headers === 'object')
+            ? { ...currentOtel.headers }
+            : {};
+        if (token)
+            headers.Authorization = `Bearer ${token}`;
+        const nextOtel = {
+            ...currentOtel,
+            enabled: true,
+            endpoint,
+            protocol: 'http/protobuf',
+            headers,
+            serviceName: String(currentOtel.serviceName || 'openclaw'),
+            traces: false,
+            metrics: true,
+            logs: false,
+            flushIntervalMs: Number(currentOtel.flushIntervalMs) > 0 ? Number(currentOtel.flushIntervalMs) : 5000,
+        };
+        const next = {
+            ...parsed,
+            diagnostics: {
+                ...diagnostics,
+                enabled: true,
+                otel: nextOtel,
+            },
+        };
+        if (JSON.stringify(parsed) === JSON.stringify(next))
+            return;
+        fs.writeFileSync(configPath, JSON.stringify(next, null, 2), 'utf8');
+        console.log(`[openclaw-observability] diagnostics.otel bootstrap enabled -> ${endpoint}/v1/metrics`);
+    }
+    catch (e) {
+        console.warn('[openclaw-observability] Failed to bootstrap diagnostics.otel:', e.message);
+    }
+}
+function persistScopeTokenToOpenclawConfig(params) {
+    const { configPath, scopeId, scopeToken } = params;
+    try {
+        if (!configPath || !scopeToken)
+            return;
+        if (!fs.existsSync(configPath))
+            return;
+        const raw = fs.readFileSync(configPath, 'utf8');
+        const parsed = JSON.parse(raw);
+        const plugins = (parsed.plugins && typeof parsed.plugins === 'object') ? parsed.plugins : {};
+        const entries = (plugins.entries && typeof plugins.entries === 'object') ? plugins.entries : {};
+        const obs = (entries['openclaw-observability'] && typeof entries['openclaw-observability'] === 'object')
+            ? entries['openclaw-observability']
+            : { enabled: true, config: {} };
+        const cfg = (obs.config && typeof obs.config === 'object') ? obs.config : {};
+        if (String(cfg.scopeToken || '') === scopeToken && String(cfg.scopeId || '') === scopeId)
+            return;
+        cfg.scopeId = scopeId;
+        cfg.scopeToken = scopeToken;
+        obs.config = cfg;
+        entries['openclaw-observability'] = obs;
+        plugins.entries = entries;
+        parsed.plugins = plugins;
+        fs.writeFileSync(configPath, JSON.stringify(parsed, null, 2), 'utf8');
+    }
+    catch (e) {
+        console.warn('[openclaw-observability] Failed to persist scope token to openclaw.json:', e.message);
+    }
+}
 /* ------------------------------------------------------------------ */
 /*  Fetch interceptor — inject stream_options to capture token usage   */
 /* ------------------------------------------------------------------ */
@@ -237,7 +333,7 @@ function resolveSessionTranscriptPath(sessionId) {
         return sessionTranscriptPathCache.get(sessionId) ?? null;
     }
     try {
-        const agentsRoot = path.join(os.homedir(), '.openclaw', 'agents');
+        const agentsRoot = path.join(resolveOpenclawStateDir(), 'agents');
         const agentDirs = fs.readdirSync(agentsRoot, { withFileTypes: true });
         for (const dir of agentDirs) {
             if (!dir.isDirectory())
@@ -364,6 +460,147 @@ function makeToolTimingKey(sessionId, runId, toolName, toolCallId) {
         return `tc:id:${toolCallId}`;
     return `tc:fallback:${sessionId}:${runId ?? 'norun'}:${toolName}`;
 }
+function toObject(value) {
+    if (value && typeof value === 'object' && !Array.isArray(value))
+        return value;
+    return {};
+}
+function pickFirstString(...values) {
+    for (const v of values) {
+        if (typeof v !== 'string')
+            continue;
+        const s = v.trim();
+        if (s)
+            return s;
+    }
+    return '';
+}
+function parseJsonObjectFromText(value) {
+    if (typeof value !== 'string')
+        return null;
+    const t = value.trim();
+    if (!t || (t[0] !== '{' && t[0] !== '['))
+        return null;
+    try {
+        const parsed = JSON.parse(t);
+        return parsed && typeof parsed === 'object' ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+function getActionRunIdFromPayload(action) {
+    return pickFirstString(action.runId, toObject(action.inputParams).runId, toObject(action.outputResult).runId, toObject(action.inputParams).run_id, toObject(action.outputResult).run_id);
+}
+function parseAnnounceChildRunId(runId) {
+    if (!runId.startsWith('announce:v1:'))
+        return '';
+    const pos = runId.lastIndexOf(':');
+    if (pos < 0 || pos >= runId.length - 1)
+        return '';
+    const tail = runId.slice(pos + 1).trim();
+    return /^[0-9a-fA-F-]{16,}$/.test(tail) ? tail : '';
+}
+function extractSpawnChildRunId(action) {
+    if (action.actionName !== 'tool_call:sessions_spawn')
+        return '';
+    const out = toObject(action.outputResult);
+    const details = toObject(out.details);
+    let childRunId = pickFirstString(details.runId, details.run_id);
+    if (childRunId)
+        return childRunId;
+    const content = out.content;
+    if (!Array.isArray(content))
+        return '';
+    for (const item of content) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const block = item;
+        if (block.type !== 'text')
+            continue;
+        const parsed = parseJsonObjectFromText(block.text);
+        if (!parsed)
+            continue;
+        childRunId = pickFirstString(parsed.runId, parsed.run_id);
+        if (childRunId)
+            return childRunId;
+    }
+    return '';
+}
+function enrichRunLineage(action) {
+    const runId = getActionRunIdFromPayload(action);
+    if (runId && !action.runId)
+        action.runId = runId;
+    const effectiveRunId = action.runId || '';
+    const inputObj = toObject(action.inputParams);
+    const outputObj = toObject(action.outputResult);
+    const now = Date.now();
+    if (effectiveRunId) {
+        const link = runParentLinks.get(effectiveRunId);
+        if (link)
+            link.seenAt = now;
+    }
+    if (action.actionName === 'tool_call:sessions_spawn') {
+        const childRunId = extractSpawnChildRunId(action);
+        const toolCallId = pickFirstString(inputObj.toolCallId, outputObj.toolCallId, inputObj.callId, outputObj.callId, inputObj.tool_call_id, outputObj.tool_call_id);
+        const parentObservationId = effectiveRunId && toolCallId ? `tool:${effectiveRunId}:${toolCallId}` : '';
+        if (effectiveRunId && childRunId && childRunId !== effectiveRunId) {
+            runParentLinks.set(childRunId, {
+                parentRunId: effectiveRunId,
+                parentObservationId: parentObservationId || undefined,
+                seenAt: now,
+            });
+        }
+    }
+    if (!action.parentRunId && effectiveRunId) {
+        const announceChildRunId = parseAnnounceChildRunId(effectiveRunId);
+        if (announceChildRunId) {
+            const link = runParentLinks.get(announceChildRunId);
+            if (link?.parentRunId && link.parentRunId !== effectiveRunId) {
+                action.parentRunId = link.parentRunId;
+                if (!inputObj.parentRunId && !inputObj.parent_run_id)
+                    inputObj.parentRunId = link.parentRunId;
+                if (!outputObj.parentRunId && !outputObj.parent_run_id)
+                    outputObj.parentRunId = link.parentRunId;
+                if (!action.parentObservationId && link.parentObservationId) {
+                    action.parentObservationId = link.parentObservationId;
+                }
+                if (link.parentObservationId) {
+                    if (!inputObj.parentObservationId && !inputObj.parent_observation_id)
+                        inputObj.parentObservationId = link.parentObservationId;
+                    if (!outputObj.parentObservationId && !outputObj.parent_observation_id)
+                        outputObj.parentObservationId = link.parentObservationId;
+                }
+                link.seenAt = now;
+            }
+        }
+        else {
+            const direct = runParentLinks.get(effectiveRunId);
+            if (direct?.parentRunId && direct.parentRunId !== effectiveRunId) {
+                action.parentRunId = direct.parentRunId;
+                if (!inputObj.parentRunId && !inputObj.parent_run_id)
+                    inputObj.parentRunId = direct.parentRunId;
+                if (!outputObj.parentRunId && !outputObj.parent_run_id)
+                    outputObj.parentRunId = direct.parentRunId;
+                if (!action.parentObservationId && direct.parentObservationId) {
+                    action.parentObservationId = direct.parentObservationId;
+                }
+                if (direct.parentObservationId) {
+                    if (!inputObj.parentObservationId && !inputObj.parent_observation_id)
+                        inputObj.parentObservationId = direct.parentObservationId;
+                    if (!outputObj.parentObservationId && !outputObj.parent_observation_id)
+                        outputObj.parentObservationId = direct.parentObservationId;
+                }
+                direct.seenAt = now;
+            }
+        }
+    }
+    if (Object.keys(inputObj).length > 0)
+        action.inputParams = inputObj;
+    if (Object.keys(outputObj).length > 0)
+        action.outputResult = outputObj;
+    return action;
+}
 /**
  * Periodically clean up stale Map entries to prevent memory leaks when llm_input fires but llm_output does not
  */
@@ -418,6 +655,17 @@ function cleanupStaleMaps() {
             replayedAssistantMessages.delete(key);
         }
     }
+    for (const [childRunId, link] of runParentLinks) {
+        if (!link || now - link.seenAt > MAP_ENTRY_TTL_MS) {
+            runParentLinks.delete(childRunId);
+        }
+    }
+    if (runParentLinks.size > MAX_MAP_ENTRIES) {
+        const entries = Array.from(runParentLinks.entries()).sort((a, b) => a[1].seenAt - b[1].seenAt);
+        const drop = entries.length - MAX_MAP_ENTRIES;
+        for (let i = 0; i < drop; i++)
+            runParentLinks.delete(entries[i][0]);
+    }
 }
 /**
  * Extract image/media metadata from historyMessages
@@ -461,8 +709,8 @@ function extractMediaMeta(historyMessages) {
 const sessionStatsMap = new Map();
 /** Max session stats entries */
 const MAX_SESSION_STATS = 200;
-/** Update session statistics and return AuditSession */
-function updateSessionStats(sessionId, modelName, userId, tokens) {
+/** Update session statistics and return ObservabilitySession */
+function updateSessionStats(sessionId, modelName, userId, parentSessionId, tokens, organizationId, scopeId, environment) {
     const sctx = (0, runtime_1.getSessionCtx)(sessionId);
     let stats = sessionStatsMap.get(sessionId);
     if (!stats) {
@@ -471,7 +719,11 @@ function updateSessionStats(sessionId, modelName, userId, tokens) {
             lastSeen: new Date(),
             modelName,
             userId,
+            parentSessionId,
             channelId: sctx.channelId,
+            organizationId,
+            scopeId,
+            environment,
             totalActions: 0,
             totalTokens: 0,
         };
@@ -484,22 +736,41 @@ function updateSessionStats(sessionId, modelName, userId, tokens) {
         stats.modelName = modelName;
     if (userId)
         stats.userId = userId;
+    if (parentSessionId)
+        stats.parentSessionId = parentSessionId;
+    if (organizationId)
+        stats.organizationId = organizationId;
+    if (scopeId)
+        stats.scopeId = scopeId;
+    if (environment)
+        stats.environment = environment;
     if (sctx.channelId)
         stats.channelId = sctx.channelId;
+    if (sctx.parentSessionId)
+        stats.parentSessionId = sctx.parentSessionId;
     return {
         sessionId,
         userId: stats.userId,
+        parentSessionId: stats.parentSessionId,
         modelName: stats.modelName,
         channelId: stats.channelId,
+        organizationId: stats.organizationId,
+        scopeId: stats.scopeId,
+        environment: stats.environment,
         startTime: stats.firstSeen,
         endTime: stats.lastSeen,
         totalActions: stats.totalActions,
         totalTokens: stats.totalTokens,
     };
 }
-/** Helper to build an AuditAction (auto-fills defaults) */
+/** Helper to build an ObservabilityAction (auto-fills defaults) */
 function makeAction(sessionId, overrides) {
     const sctx = (0, runtime_1.getSessionCtx)(sessionId);
+    const inputObj = toObject(overrides.inputParams);
+    const outputObj = toObject(overrides.outputResult);
+    const runId = pickFirstString(overrides.runId, inputObj.runId, outputObj.runId, inputObj.run_id, outputObj.run_id);
+    const parentRunId = pickFirstString(overrides.parentRunId, inputObj.parentRunId, outputObj.parentRunId, inputObj.parent_run_id, outputObj.parent_run_id);
+    const parentObservationId = pickFirstString(overrides.parentObservationId, inputObj.parentObservationId, outputObj.parentObservationId, inputObj.parent_observation_id, outputObj.parent_observation_id);
     return {
         sessionId,
         actionType: overrides.actionType,
@@ -511,7 +782,14 @@ function makeAction(sessionId, overrides) {
         completionTokens: overrides.completionTokens ?? null,
         durationMs: overrides.durationMs ?? null,
         userId: overrides.userId ?? sctx.userId,
+        parentSessionId: overrides.parentSessionId ?? sctx.parentSessionId,
+        runId: runId || undefined,
+        parentRunId: parentRunId || undefined,
+        parentObservationId: parentObservationId || undefined,
         channelId: overrides.channelId ?? sctx.channelId,
+        organizationId: overrides.organizationId ?? _defaultTenantScope.organizationId,
+        scopeId: overrides.scopeId ?? _defaultTenantScope.scopeId,
+        environment: overrides.environment ?? _defaultTenantScope.environment,
         createdAt: overrides.createdAt ?? new Date(),
     };
 }
@@ -534,9 +812,23 @@ let _mapCleanupTimer = null;
 let _checkpointTimer = null;
 let _metricsCleanupTimer = null;
 let _unsubscribeAgentEvents = null;
+let _defaultTenantScope = {
+    organizationId: 'local',
+    scopeId: 'local',
+    environment: 'prod',
+};
 function activate(api) {
     const rawConfig = (api.pluginConfig || api.config || {});
     const config = (0, config_1.resolveConfig)(rawConfig);
+    const stateDir = resolveOpenclawStateDir();
+    const openclawConfigPath = path.join(stateDir, 'openclaw.json');
+    const effectiveScopeId = String(process.env.OPENCLAW_OBSERVABILITY_SCOPE_ID || config.scopeId || 'local');
+    const effectiveScopeToken = String(process.env.OPENCLAW_OBSERVABILITY_SCOPE_TOKEN || config.scopeToken || '');
+    _defaultTenantScope = {
+        organizationId: String(process.env.OPENCLAW_OBSERVABILITY_ORG_ID || config.organizationId || 'local'),
+        scopeId: effectiveScopeId,
+        environment: String(process.env.OPENCLAW_OBSERVABILITY_ENV || config.environment || 'prod'),
+    };
     const logInfo = (message) => {
         if (typeof api.logger?.info === 'function') {
             api.logger.info(message);
@@ -548,10 +840,26 @@ function activate(api) {
     const isFirstActivation = !_writer;
     if (!_writer) {
         if (config.mode === 'remote') {
-            _writer = new mysql_writer_1.MySQLWriter(config.mysql);
+            _writer = new mysql_writer_1.MySQLWriter(config.mysql, {
+                scopeId: effectiveScopeId,
+                scopeToken: effectiveScopeToken,
+                dataRetentionDays: config.dataRetentionDays,
+                onTokenIssued: (token) => {
+                    persistScopeTokenToOpenclawConfig({
+                        configPath: openclawConfigPath,
+                        scopeId: effectiveScopeId,
+                        scopeToken: token,
+                    });
+                },
+            });
+        }
+        else if (config.mode === 'cloud') {
+            _writer = new cloud_export_writer_1.CloudExportWriter(config.cloud);
         }
         else {
-            _writer = new duckdb_local_writer_1.DuckDBLocalWriter(config.duckdb);
+            _writer = new duckdb_local_writer_1.DuckDBLocalWriter(config.duckdb, {
+                dataRetentionDays: config.dataRetentionDays,
+            });
         }
     }
     if (!_buffer) {
@@ -570,7 +878,6 @@ function activate(api) {
         (0, runtime_1.installFetchInterceptor)();
     }
     const redactor = new redaction_1.Redactor(config.redaction);
-    const stateDir = path.join(os.homedir(), '.openclaw');
     const metricsRetentionDaysRaw = Number(config.metrics.retentionDays);
     const metricsRetentionDays = Number.isFinite(metricsRetentionDaysRaw)
         ? Math.max(0, Math.floor(metricsRetentionDaysRaw))
@@ -657,6 +964,14 @@ function activate(api) {
             metricsCleanupRunning = false;
         }
     }
+    async function runStartupMaintenance() {
+        if (typeof writer.runStartupMaintenance !== 'function')
+            return;
+        const ready = await writer.ensureReady();
+        if (!ready)
+            return;
+        await writer.runStartupMaintenance();
+    }
     function startServices() {
         if (_servicesStarted)
             return;
@@ -700,12 +1015,29 @@ function activate(api) {
     }
     // Lazy initialization: database file will be created on first write or first API query
     // This avoids creating empty database files when plugin is installed but never used
-    const dbConfigured = config.mode === 'local'
-        || (config.mysql.host !== 'localhost' || config.mysql.password !== '');
+    const dbConfigured = config.mode === 'cloud'
+        ? Boolean(config.cloud.endpoint && config.cloud.apiKey)
+        : (config.mode === 'local'
+            || (config.mysql.host !== 'localhost' || config.mysql.password !== ''));
     const dbStatus = dbConfigured ? 'ready' : 'not-configured';
+    // Pre-warm database initialization as early as possible to reduce
+    // first-request latency after gateway/plugin startup.
+    if (dbConfigured) {
+        writer.ensureReady()
+            .then((ok) => {
+            if (ok) {
+                void runStartupMaintenance().catch((err) => {
+                    console.warn('[openclaw-observability] Startup maintenance failed:', err.message);
+                });
+                startServices();
+            }
+        })
+            .catch((err) => {
+            console.warn('[openclaw-observability] Early DB warmup failed (will retry lazily):', err.message);
+        });
+    }
     // UI shell stays directly open; legacy HTTP APIs still accept gateway token
     // for backward compatibility. WS data access is authenticated at gateway connect.
-    const openclawConfigPath = path.join(stateDir, 'openclaw.json');
     let observabilityToken;
     try {
         if (fs.existsSync(openclawConfigPath)) {
@@ -726,12 +1058,24 @@ function activate(api) {
     catch (e) {
         console.warn('[openclaw-observability] Failed to read gateway token from config:', e.message);
     }
+    ensureDiagnosticsOtelBootstrap({
+        configPath: openclawConfigPath,
+        otlpPath: config.metrics.otlpPath,
+        token: observabilityToken,
+    });
     const tokenCompatEnabled = Boolean(observabilityToken);
     // Always register HTTP routes on the current api (the latest api serves HTTP)
     const registerHttpRoute = api.registerHttpRoute;
     const httpRouteEnabled = typeof registerHttpRoute === 'function';
     if (httpRouteEnabled) {
-        (0, routes_1.registerAuditRoutes)(registerHttpRoute.bind(api), writer, observabilityToken, {
+        const runReplay = async (params) => {
+            return (0, replay_runtime_1.runReplayRequestViaRuntime)({
+                configPath: openclawConfigPath,
+                runtime: api.runtime,
+                ...params,
+            });
+        };
+        (0, routes_1.registerObservabilityRoutes)(registerHttpRoute.bind(api), writer, observabilityToken, {
             getConfig: () => securityScanner.getConfig(),
             updateConfig: (patch) => {
                 const next = securityScanner.updateConfig(patch);
@@ -742,7 +1086,18 @@ function activate(api) {
                     `customRegexRules=${next.customRegexRules.length}`);
                 return next;
             },
-        }, config.metrics);
+        }, config.metrics, {
+            showTenantScope: config.mode !== 'local',
+        }, {
+            runReplay,
+        }, {
+            enabled: config.mode === 'cloud' || config.cloud.enabled,
+            secretSalt: process.env.OPENCLAW_OBSERVABILITY_API_KEY_SALT || '',
+        }, {
+            openclawRootDir: stateDir,
+            openclawConfigPath,
+            workspaceMediaRoot: path.join(stateDir, 'workspace', 'media'),
+        });
     }
     else {
         console.warn('[openclaw-observability] registerHttpRoute not available — observability UI disabled');
@@ -756,6 +1111,8 @@ function activate(api) {
             config,
             securityScanner,
             persistSecurityConfig,
+            openclawConfigPath,
+            runtime: api.runtime,
         });
     }
     else {
@@ -764,12 +1121,13 @@ function activate(api) {
     // ====================== Helper functions ======================
     /** Record action, update session stats, and run security scan */
     function recordAction(action, tokens = 0) {
-        void buffer.addAction(action);
-        const session = updateSessionStats(action.sessionId, action.modelName, action.userId, tokens);
+        const enriched = enrichRunLineage(action);
+        void buffer.addAction(enriched);
+        const session = updateSessionStats(enriched.sessionId, enriched.modelName, enriched.userId, enriched.parentSessionId || '', tokens, enriched.organizationId || _defaultTenantScope.organizationId, enriched.scopeId || _defaultTenantScope.scopeId, enriched.environment || _defaultTenantScope.environment);
         void buffer.addSession(session);
         // --- Security scan ---
         try {
-            const alerts = securityScanner.scan(action);
+            const alerts = securityScanner.scan(enriched);
             if (alerts.length > 0) {
                 // Prevent _alertBuffer from growing unbounded
                 if (_alertBuffer.length + alerts.length > MAX_ALERT_BUFFER) {
@@ -953,52 +1311,8 @@ function activate(api) {
     // =====================================================================
     //  1. Agent lifecycle
     // =====================================================================
-    // before_model_resolve: before model selection
-    api.on('before_model_resolve', (event, ctx) => {
-        try {
-            const e = event;
-            const c = ctx;
-            const sid = (0, runtime_1.resolveSessionId)(undefined, c?.sessionId);
-            const sctx = (0, runtime_1.getSessionCtx)(sid);
-            if (c?.agentId)
-                sctx.userId = c.agentId;
-            recordAction(makeAction(sid, {
-                actionType: types_1.ActionType.ModelResolve,
-                actionName: 'before_model_resolve',
-                userId: c?.agentId,
-                inputParams: redactor.redact({ prompt: e.prompt }),
-            }));
-            console.log(`[openclaw-observability] before_model_resolve: session=${sid}`);
-        }
-        catch (err) {
-            console.error('[openclaw-observability] Error in before_model_resolve:', err);
-        }
-    });
-    // before_prompt_build: before prompt construction
-    api.on('before_prompt_build', (event, ctx) => {
-        try {
-            const e = event;
-            const c = ctx;
-            const sid = (0, runtime_1.resolveSessionId)(undefined, c?.sessionId);
-            const summary = summarizePromptMessages(e.messages);
-            recordAction(makeAction(sid, {
-                actionType: types_1.ActionType.PromptBuild,
-                actionName: 'before_prompt_build',
-                userId: c?.agentId,
-                inputParams: redactor.redact({
-                    prompt: e.prompt,
-                    messageCount: summary.messageCount,
-                    messageRoleCounts: summary.roleCounts,
-                    firstMessageRoles: summary.firstRoles,
-                    messages: (0, runtime_1.sanitizePayloadForStorage)(e.messages),
-                }),
-            }));
-            console.log(`[openclaw-observability] before_prompt_build: session=${sid} msgs=${Array.isArray(e.messages) ? e.messages.length : '?'}`);
-        }
-        catch (err) {
-            console.error('[openclaw-observability] Error in before_prompt_build:', err);
-        }
-    });
+    // before_model_resolve / before_prompt_build intentionally not collected:
+    // these hooks are high-frequency and overlap with llm_input/llm_output data.
     // before_agent_start: skipped — legacy hook that fires twice
     // and fully overlaps with before_model_resolve + before_prompt_build
     // agent_end: Agent run finished
@@ -1006,17 +1320,25 @@ function activate(api) {
         try {
             const e = event;
             const c = ctx;
-            const sid = (0, runtime_1.resolveSessionId)(undefined, c?.sessionId);
+            const sid = (0, runtime_1.resolveSessionIdWithRun)({
+                ctxSessionId: c?.sessionId,
+                runId: e.runId || c?.runId,
+                allowGlobalFallback: false,
+                runSessionIds,
+            });
+            if (sid === 'unknown')
+                return;
             recordAction(makeAction(sid, {
                 actionType: types_1.ActionType.AgentEnd,
                 actionName: 'agent_end',
-                userId: c?.agentId,
                 outputResult: redactor.redact({
                     success: e.success,
                     error: e.error,
                     messageCount: Array.isArray(e.messages) ? e.messages.length : 0,
+                    runId: e.runId || c?.runId,
                 }),
                 durationMs: e.durationMs ?? null,
+                runId: e.runId || c?.runId,
             }));
             console.log(`[openclaw-observability] agent_end: session=${sid} success=${e.success} duration=${e.durationMs}ms`);
         }
@@ -1033,6 +1355,8 @@ function activate(api) {
             const e = event;
             const c = ctx;
             const sid = (0, runtime_1.resolveSessionId)(e.sessionId, c?.sessionId);
+            if (sid === 'unknown')
+                return;
             const sctx = (0, runtime_1.getSessionCtx)(sid);
             // Parse image/media metadata from historyMessages
             const media = extractMediaMeta(e.historyMessages ?? []);
@@ -1065,7 +1389,6 @@ function activate(api) {
                             inputParams: redactor.redact(replayMeta),
                             outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
                             createdAt: new Date(replayAt.getTime() + 1),
-                            userId: c?.agentId,
                         }));
                     }
                 }
@@ -1076,6 +1399,15 @@ function activate(api) {
                     prompt: e.prompt,
                 });
                 const sanitizedHistory = (0, runtime_1.sanitizePayloadForStorage)(e.historyMessages);
+                const isReplayRun = ((typeof e.runId === 'string' && e.runId.startsWith('replay:'))
+                    || sid.startsWith('replay-'));
+                const replayMeta = isReplayRun
+                    ? {
+                        isReplay: true,
+                        replaySessionId: sid,
+                        replayRunId: e.runId,
+                    }
+                    : undefined;
                 llmRunStartTimes.set(e.runId, Date.now());
                 runSessionIds.set(e.runId, sid);
                 runInputs.set(e.runId, {
@@ -1085,9 +1417,10 @@ function activate(api) {
                     imagesCount: e.imagesCount,
                     media: media.length > 0 ? media : undefined,
                     historyMessages: Array.isArray(sanitizedHistory) ? sanitizedHistory : undefined,
+                    replayMeta,
                 });
             }
-            if (c?.agentId)
+            if (c?.agentId && !sctx.userId)
                 sctx.userId = c.agentId;
             // Identify channel from multiple sources
             const identifiedChannel = (0, runtime_1.identifyChannel)(c);
@@ -1121,6 +1454,8 @@ function activate(api) {
                 runInputs.delete(e.runId);
             // Token usage: multiple extraction strategies
             const sid = (0, runtime_1.resolveSessionId)(e.sessionId, c?.sessionId);
+            if (sid === 'unknown')
+                return;
             let promptTokens = e.usage?.input ?? null;
             let completionTokens = e.usage?.output ?? null;
             let cacheRead = e.usage?.cacheRead ?? null;
@@ -1188,13 +1523,17 @@ function activate(api) {
                 inputData.historyMessages = cachedInput.historyMessages;
                 inputData.historyMessageCount = cachedInput.historyMessages.length;
             }
+            if (cachedInput?.replayMeta?.isReplay) {
+                inputData.replay = cachedInput.replayMeta;
+            }
             const sctx = (0, runtime_1.getSessionCtx)(sid);
-            if (c?.agentId)
+            if (c?.agentId && !sctx.userId)
                 sctx.userId = c.agentId;
             sctx.modelName = modelName;
+            const isReplayCall = cachedInput?.replayMeta?.isReplay === true;
             recordAction(makeAction(sid, {
-                actionType: types_1.ActionType.Message,
-                actionName: `llm_call:${modelName}`,
+                actionType: isReplayCall ? types_1.ActionType.Replay : types_1.ActionType.Message,
+                actionName: isReplayCall ? `replay_call:${modelName}` : `llm_call:${modelName}`,
                 modelName,
                 inputParams: redactor.redact(inputData),
                 outputResult: redactor.redact({
@@ -1212,53 +1551,61 @@ function activate(api) {
                 promptTokens,
                 completionTokens,
                 durationMs,
-                userId: c?.agentId,
             }), totalTokens);
             const endedAt = Date.now();
             const syntheticTs = endedAt;
             if (e.runId) {
-                const transcriptToolUseMessages = extractToolUseAssistantMessagesFromTranscript({
-                    sessionId: sid,
-                    runStartedAt,
-                    runEndedAt: endedAt,
-                });
-                for (const msg of transcriptToolUseMessages) {
-                    const replayKey = `${sid}:${e.runId}:${msg.messageId}`;
-                    if (replayedAssistantMessages.has(replayKey))
-                        continue;
-                    replayedAssistantMessages.set(replayKey, Date.now());
-                    const replayMeta = {
-                        runId: e.runId,
-                        source: 'transcript_message',
-                        synthetic: true,
-                        stopReason: msg.stopReason ?? 'toolUse',
-                        messageId: msg.messageId,
-                        toolCallIds: msg.toolCallIds,
-                    };
-                    const replayAt = new Date(msg.timestampMs);
-                    if (msg.thinking.length > 0) {
-                        const thinkingText = msg.thinking.join('\n\n');
-                        recordAction(makeAction(sid, {
-                            actionType: types_1.ActionType.Thinking,
-                            actionName: 'thinking',
-                            modelName,
-                            inputParams: redactor.redact(replayMeta),
-                            outputResult: redactor.redact({ text: thinkingText, length: thinkingText.length }),
-                            createdAt: replayAt,
-                            userId: c?.agentId,
-                        }));
-                    }
-                    if (msg.texts.length > 0) {
-                        const assistantText = msg.texts.join('\n\n');
-                        recordAction(makeAction(sid, {
-                            actionType: types_1.ActionType.AssistantStream,
-                            actionName: 'assistant_stream',
-                            modelName,
-                            inputParams: redactor.redact(replayMeta),
-                            outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
-                            createdAt: new Date(msg.timestampMs + 1),
-                            userId: c?.agentId,
-                        }));
+                // Transcript replay is a fallback-only path for providers/runtimes that
+                // fail to expose assistant stream/thinking in llm_output payloads.
+                // If we already have direct content for this run, replay can pull in a
+                // neighboring toolUse message and mis-attach it to the current run.
+                const shouldReplayFromTranscript = assistantTexts.length === 0 &&
+                    fallbackArtifacts.thinking.length === 0;
+                if (!shouldReplayFromTranscript) {
+                    // skip transcript replay for this run
+                }
+                else {
+                    const transcriptToolUseMessages = extractToolUseAssistantMessagesFromTranscript({
+                        sessionId: sid,
+                        runStartedAt,
+                        runEndedAt: endedAt,
+                    });
+                    for (const msg of transcriptToolUseMessages) {
+                        const replayKey = `${sid}:${e.runId}:${msg.messageId}`;
+                        if (replayedAssistantMessages.has(replayKey))
+                            continue;
+                        replayedAssistantMessages.set(replayKey, Date.now());
+                        const replayMeta = {
+                            runId: e.runId,
+                            source: 'transcript_message',
+                            synthetic: true,
+                            stopReason: msg.stopReason ?? 'toolUse',
+                            messageId: msg.messageId,
+                            toolCallIds: msg.toolCallIds,
+                        };
+                        const replayAt = new Date(msg.timestampMs);
+                        if (msg.thinking.length > 0) {
+                            const thinkingText = msg.thinking.join('\n\n');
+                            recordAction(makeAction(sid, {
+                                actionType: types_1.ActionType.Thinking,
+                                actionName: 'thinking',
+                                modelName,
+                                inputParams: redactor.redact(replayMeta),
+                                outputResult: redactor.redact({ text: thinkingText, length: thinkingText.length }),
+                                createdAt: replayAt,
+                            }));
+                        }
+                        if (msg.texts.length > 0) {
+                            const assistantText = msg.texts.join('\n\n');
+                            recordAction(makeAction(sid, {
+                                actionType: types_1.ActionType.AssistantStream,
+                                actionName: 'assistant_stream',
+                                modelName,
+                                inputParams: redactor.redact(replayMeta),
+                                outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
+                                createdAt: new Date(msg.timestampMs + 1),
+                            }));
+                        }
                     }
                 }
                 if (assistantTexts.length > 0 &&
@@ -1271,7 +1618,6 @@ function activate(api) {
                         inputParams: redactor.redact({ runId: e.runId, stream: 'assistant', synthetic: true }),
                         outputResult: redactor.redact({ text: assistantText, length: assistantText.length }),
                         createdAt: new Date(syntheticTs - 1),
-                        userId: c?.agentId,
                     }));
                 }
                 if (fallbackArtifacts.thinking.length > 0 &&
@@ -1304,7 +1650,6 @@ function activate(api) {
                         outputResult: redactor.redact({ text: thinkingText, length: thinkingText.length }),
                         createdAt: new Date(thinkingEndAt),
                         durationMs: inferredThinkingDuration,
-                        userId: c?.agentId,
                     }));
                 }
             }
@@ -1333,10 +1678,11 @@ function activate(api) {
             const e = event;
             const c = ctx;
             const sid = (0, runtime_1.resolveSessionId)(undefined, c?.sessionId);
+            if (sid === 'unknown')
+                return;
             recordAction(makeAction(sid, {
                 actionType: types_1.ActionType.Compaction,
                 actionName: 'before_compaction',
-                userId: c?.agentId,
                 inputParams: redactor.redact({
                     messageCount: e.messageCount,
                     compactingCount: e.compactingCount,
@@ -1357,10 +1703,11 @@ function activate(api) {
             const e = event;
             const c = ctx;
             const sid = (0, runtime_1.resolveSessionId)(undefined, c?.sessionId);
+            if (sid === 'unknown')
+                return;
             recordAction(makeAction(sid, {
                 actionType: types_1.ActionType.Compaction,
                 actionName: 'after_compaction',
-                userId: c?.agentId,
                 outputResult: redactor.redact({
                     messageCount: e.messageCount,
                     compactedCount: e.compactedCount,
@@ -1380,10 +1727,11 @@ function activate(api) {
             const e = event;
             const c = ctx;
             const sid = (0, runtime_1.resolveSessionId)(undefined, c?.sessionId);
+            if (sid === 'unknown')
+                return;
             recordAction(makeAction(sid, {
                 actionType: types_1.ActionType.Reset,
                 actionName: 'before_reset',
-                userId: c?.agentId,
                 inputParams: redactor.redact({
                     reason: e.reason,
                     messageCount: Array.isArray(e.messages) ? e.messages.length : 0,
@@ -1407,6 +1755,7 @@ function activate(api) {
         recordAction,
         identifyChannel: runtime_1.identifyChannel,
         saveChannelToContext: runtime_1.saveChannelToContext,
+        saveUserToContext: runtime_1.saveUserToContext,
         resolveSessionIdWithRun: (args) => (0, runtime_1.resolveSessionIdWithRun)({ ...args, runSessionIds }),
     });
     // =====================================================================
@@ -1448,6 +1797,7 @@ function activate(api) {
         redactor,
         makeAction,
         recordAction,
+        bindParentSession: runtime_1.bindParentSession,
         resolveSessionIdWithRun: (args) => (0, runtime_1.resolveSessionIdWithRun)({ ...args, runSessionIds }),
     });
     // =====================================================================
@@ -1460,7 +1810,9 @@ function activate(api) {
             // Pre-initialize DuckDB during startup (not lazily on first request).
             // This ensures WAL recovery completes before any user requests arrive,
             // preventing 503 errors on the first request after restart.
-            writer.ensureReady().catch((err) => {
+            writer.ensureReady()
+                .then(() => runStartupMaintenance())
+                .catch((err) => {
                 console.error('[openclaw-observability] Pre-init DuckDB failed (will retry on first write):', err);
             });
             recordAction(makeAction('system', {
@@ -1497,7 +1849,9 @@ function activate(api) {
     });
     const dbTarget = config.mode === 'remote'
         ? `${config.mysql.host}/${config.mysql.database}`
-        : config.duckdb.path;
+        : config.mode === 'cloud'
+            ? config.cloud.endpoint
+            : config.duckdb.path;
     logInfo(`[Observability] Plugin activated (mode: ${config.mode}, db: ${dbStatus}:${dbTarget}, ui: ${httpRouteEnabled ? '/plugins/observability/' : 'disabled'}, metrics: ${config.metrics.enabled ? `${config.metrics.otlpPath}/v1/metrics` : 'disabled'}, metricsRetentionDays: ${metricsRetentionDays > 0 ? metricsRetentionDays : 'off'}, rpc: ${gatewayRpcEnabled ? 'enabled' : 'disabled'}, runtimeEvents: ${runtimeEventsStatus}, tokenCompat: ${tokenCompatEnabled ? 'on' : 'off'})`);
     // =====================================================================
     //  Return deactivate function for cleanup