@tloncorp/openclaw 0.6.0 → 0.6.1

package/dist/index.js

@@ -2,20 +2,24 @@ import { spawn } from 'node:child_process';
 import { createRequire } from 'node:module';
 import { dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { defineChannelPluginEntry } from 'openclaw/plugin-sdk/core';
+import { defineChannelPluginEntry, } from 'openclaw/plugin-sdk/core';
+import { onDiagnosticEvent, onInternalDiagnosticEvent, } from 'openclaw/plugin-sdk/diagnostic-runtime';
 import { tlonPlugin } from './src/channel.js';
+import { installTlonDiagnosticSubscriptions, shouldInstallTlonDiagnosticSubscriptions, } from './src/diagnostic-subscriptions.js';
 import { sendGatewayStop } from './src/gateway-status.js';
 import { createGatewayStatusManager, setGatewayStatusManager, } from './src/gateway-status.js';
 import { resolveBridgeForCommand } from './src/monitor/command-auth.js';
+import { isRouteDebugEnabled } from './src/monitor/session-routing.js';
 import { handleOwnerListenCommand } from './src/owner-listen-command.js';
 import { setTlonRuntime } from './src/runtime.js';
 import { getSessionRole } from './src/session-roles.js';
-import { recordToolCall } from './src/telemetry.js';
+import { parseTlonTarget } from './src/targets.js';
+import { formatTlonTelemetryErrorText, recordToolCall, reportHarnessError, reportOutboundRoute, reportPluginError, reportSessionDiagnostic, reportSessionLifecycle, reportSessionTurnCreated, reportTelemetryError, } from './src/telemetry.js';
 import { resolveTlonBinary } from './src/tlon-binary.js';
 import { checkBlockedSendOperation } from './src/tlon-tool-guard.js';
 import { formatToolTraceEvent, liveToolTraceContentsEnabled, shouldLogAfterToolTrace, } from './src/tool-trace.js';
 import { listTlonAccountIds, resolveTlonAccount } from './src/types.js';
-import { PLUGIN_COMMIT, PLUGIN_VERSION } from './src/version.generated.js';
+import { formatTlonVersionIdentity, resolveTlonSkillVersion, setTlonSkillVersionResolver, } from './src/version.js';
 export { tlonPlugin } from './src/channel.js';
 export { setTlonRuntime } from './src/runtime.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -115,7 +119,7 @@ function shellSplit(str) {
 /**
  * Run the tlon command and return the result
  */
-function runTlonCommand(binary, args, credentials) {
+function runTlonCommand(binary, args, credentials, options) {
     return new Promise((resolve, reject) => {
         const env = { ...process.env };
         if (credentials) {
@@ -126,6 +130,15 @@ function runTlonCommand(binary, args, credentials) {
         const child = spawn(binary, args, { env });
         let stdout = '';
         let stderr = '';
+        let timedOut = false;
+        let timeout;
+        const timeoutMs = options?.timeoutMs;
+        const cleanup = () => {
+            if (timeout) {
+                clearTimeout(timeout);
+                timeout = undefined;
+            }
+        };
         child.stdout.on('data', (data) => {
             stdout += data.toString();
         });
@@ -133,10 +146,21 @@ function runTlonCommand(binary, args, credentials) {
             stderr += data.toString();
         });
         child.on('error', (err) => {
+            cleanup();
             reject(new Error(`Failed to run tlon: ${err.message}`));
         });
+        if (timeoutMs) {
+            timeout = setTimeout(() => {
+                timedOut = true;
+                child.kill('SIGTERM');
+            }, timeoutMs);
+        }
         child.on('close', (code) => {
-            if (code !== 0) {
+            cleanup();
+            if (timedOut) {
+                reject(new Error(`tlon timed out after ${timeoutMs}ms`));
+            }
+            else if (code !== 0) {
                 reject(new Error(stderr || `tlon exited with code ${code}`));
             }
             else {
@@ -145,6 +169,365 @@ function runTlonCommand(binary, args, credentials) {
         });
     });
 }
+function firstLine(value) {
+    return value.trim().split(/\r?\n/)[0]?.trim() || 'unknown';
+}
+function summarizeError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return firstLine(message).slice(0, 180);
+}
+async function readTlonSkillVersion(binary) {
+    try {
+        return firstLine(await runTlonCommand(binary, ['--version'], undefined, {
+            timeoutMs: 5_000,
+        }));
+    }
+    catch (error) {
+        return `unavailable (${summarizeError(error)})`;
+    }
+}
+function isTlonSessionDiagnosticEvent(event) {
+    return (event.type === 'session.stalled' ||
+        event.type === 'session.stuck' ||
+        event.type === 'session.recovery.requested' ||
+        event.type === 'session.recovery.completed');
+}
+function stringField(event, key) {
+    const value = event[key];
+    return typeof value === 'string' && value.trim() ? value : null;
+}
+function numberField(event, key) {
+    const value = event[key];
+    return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+function diagnosticErrorText(event) {
+    return stringField(event, 'error') ?? stringField(event, 'message');
+}
+function stringListField(event, key) {
+    const value = event[key];
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return value
+        .map((entry) => (typeof entry === 'string' ? entry.trim() : ''))
+        .filter(Boolean);
+}
+function diagnosticSummary(parts) {
+    return parts
+        .filter(([, value]) => value !== null && value !== undefined && value !== '')
+        .map(([key, value]) => `${key}=${String(value)}`)
+        .join(' ');
+}
+function reportHarnessDiagnostic(event) {
+    const type = stringField(event, 'type');
+    if (!type) {
+        return;
+    }
+    if (type === 'session.turn.created') {
+        reportSessionTurnCreated({
+            type,
+            sessionKey: stringField(event, 'sessionKey'),
+            sessionId: stringField(event, 'sessionId'),
+            runId: stringField(event, 'runId'),
+            agentId: stringField(event, 'agentId'),
+        });
+        return;
+    }
+    const common = {
+        harnessEventType: type,
+        sessionKey: stringField(event, 'sessionKey'),
+        sessionId: stringField(event, 'sessionId'),
+        runId: stringField(event, 'runId'),
+        agentId: stringField(event, 'agentId'),
+        provider: stringField(event, 'provider'),
+        model: stringField(event, 'model'),
+        phase: stringField(event, 'phase'),
+        outcome: stringField(event, 'outcome'),
+        errorCategory: stringField(event, 'errorCategory'),
+        failureKind: stringField(event, 'failureKind'),
+        durationMs: numberField(event, 'durationMs'),
+        errorText: diagnosticErrorText(event),
+    };
+    switch (type) {
+        case 'harness.run.error':
+            reportHarnessError({
+                ...common,
+                errorScope: 'harness',
+            });
+            return;
+        case 'harness.run.completed':
+            if (common.outcome === 'completed') {
+                return;
+            }
+            reportHarnessError({
+                ...common,
+                errorScope: 'harness',
+            });
+            return;
+        case 'model.call.error':
+            reportHarnessError({
+                ...common,
+                errorScope: 'model',
+            });
+            return;
+        case 'model.failover': {
+            const reason = stringField(event, 'reason');
+            const fromProvider = stringField(event, 'fromProvider');
+            const fromModel = stringField(event, 'fromModel');
+            const toProvider = stringField(event, 'toProvider');
+            const toModel = stringField(event, 'toModel');
+            reportHarnessError({
+                ...common,
+                errorScope: 'model',
+                provider: fromProvider,
+                model: fromModel,
+                phase: stringField(event, 'lane'),
+                outcome: 'failover',
+                errorCategory: 'model_failover',
+                failureKind: reason,
+                errorText: diagnosticSummary([
+                    ['fromProvider', fromProvider],
+                    ['fromModel', fromModel],
+                    ['toProvider', toProvider],
+                    ['toModel', toModel],
+                    ['reason', reason],
+                    ['cascadeDepth', numberField(event, 'cascadeDepth')],
+                ]),
+            });
+            return;
+        }
+        case 'tool.execution.error':
+            reportHarnessError({
+                ...common,
+                errorScope: 'tool',
+                toolName: stringField(event, 'toolName'),
+            });
+            return;
+        case 'tool.execution.blocked': {
+            const deniedReason = stringField(event, 'deniedReason');
+            const reason = stringField(event, 'reason');
+            reportHarnessError({
+                ...common,
+                errorScope: 'tool',
+                toolName: stringField(event, 'toolName'),
+                phase: stringField(event, 'toolSource'),
+                outcome: 'blocked',
+                errorCategory: 'tool_blocked',
+                failureKind: deniedReason,
+                errorText: reason ?? deniedReason,
+            });
+            return;
+        }
+        case 'tool.loop': {
+            const level = stringField(event, 'level');
+            const action = stringField(event, 'action');
+            if (level !== 'critical' && action !== 'block') {
+                return;
+            }
+            reportHarnessError({
+                ...common,
+                errorScope: 'tool',
+                toolName: stringField(event, 'toolName'),
+                phase: level,
+                outcome: action,
+                errorCategory: 'tool_loop',
+                failureKind: stringField(event, 'detector'),
+                errorText: stringField(event, 'message') ??
+                    diagnosticSummary([
+                        ['level', level],
+                        ['action', action],
+                        ['detector', stringField(event, 'detector')],
+                        ['count', numberField(event, 'count')],
+                    ]),
+            });
+            return;
+        }
+        case 'run.completed':
+            if (common.outcome === 'completed') {
+                return;
+            }
+            reportHarnessError({
+                ...common,
+                errorScope: 'run',
+            });
+            return;
+        case 'message.delivery.error':
+            reportHarnessError({
+                ...common,
+                errorScope: 'message_delivery',
+                phase: stringField(event, 'deliveryKind'),
+            });
+            return;
+        case 'message.dispatch.completed':
+            if (common.outcome !== 'error') {
+                return;
+            }
+            reportHarnessError({
+                ...common,
+                errorScope: 'message_dispatch',
+                phase: stringField(event, 'source'),
+            });
+            return;
+        case 'message.processed':
+            if (common.outcome !== 'error') {
+                return;
+            }
+            reportHarnessError({
+                ...common,
+                errorScope: 'message_processing',
+                phase: stringField(event, 'channel'),
+            });
+            return;
+        case 'diagnostic.async_queue.dropped':
+            reportHarnessError({
+                ...common,
+                errorScope: 'diagnostics',
+                outcome: 'dropped',
+                errorCategory: 'diagnostic_async_queue_dropped',
+                failureKind: 'queue_full',
+                errorText: diagnosticSummary([
+                    ['droppedEvents', numberField(event, 'droppedEvents')],
+                    ['droppedTrustedEvents', numberField(event, 'droppedTrustedEvents')],
+                    [
+                        'droppedUntrustedEvents',
+                        numberField(event, 'droppedUntrustedEvents'),
+                    ],
+                    ['queueLength', numberField(event, 'queueLength')],
+                    ['maxQueueLength', numberField(event, 'maxQueueLength')],
+                ]),
+            });
+            return;
+        case 'diagnostic.liveness.warning': {
+            const reasons = stringListField(event, 'reasons');
+            reportHarnessError({
+                ...common,
+                errorScope: 'runtime',
+                phase: stringField(event, 'phase'),
+                outcome: 'warning',
+                errorCategory: 'liveness_warning',
+                failureKind: reasons.join(',') || null,
+                durationMs: numberField(event, 'intervalMs'),
+                errorText: diagnosticSummary([
+                    ['reasons', reasons.join(',')],
+                    ['eventLoopDelayP99Ms', numberField(event, 'eventLoopDelayP99Ms')],
+                    ['eventLoopDelayMaxMs', numberField(event, 'eventLoopDelayMaxMs')],
+                    ['cpuCoreRatio', numberField(event, 'cpuCoreRatio')],
+                    ['active', numberField(event, 'active')],
+                    ['waiting', numberField(event, 'waiting')],
+                    ['queued', numberField(event, 'queued')],
+                ]),
+            });
+            return;
+        }
+        case 'diagnostic.memory.pressure': {
+            const memory = event.memory;
+            const memoryNumber = (key) => {
+                const value = memory?.[key];
+                return typeof value === 'number' && Number.isFinite(value)
+                    ? value
+                    : null;
+            };
+            reportHarnessError({
+                ...common,
+                errorScope: 'runtime',
+                outcome: stringField(event, 'level'),
+                errorCategory: 'memory_pressure',
+                failureKind: stringField(event, 'reason'),
+                durationMs: numberField(event, 'windowMs'),
+                errorText: diagnosticSummary([
+                    ['level', stringField(event, 'level')],
+                    ['reason', stringField(event, 'reason')],
+                    ['rssBytes', memoryNumber('rssBytes')],
+                    ['heapUsedBytes', memoryNumber('heapUsedBytes')],
+                    ['thresholdBytes', numberField(event, 'thresholdBytes')],
+                    ['rssGrowthBytes', numberField(event, 'rssGrowthBytes')],
+                ]),
+            });
+            return;
+        }
+        case 'payload.large':
+            if (stringField(event, 'action') !== 'rejected') {
+                return;
+            }
+            reportHarnessError({
+                ...common,
+                errorScope: 'payload',
+                phase: stringField(event, 'surface'),
+                outcome: 'rejected',
+                errorCategory: 'payload_large',
+                failureKind: stringField(event, 'reason'),
+                errorText: diagnosticSummary([
+                    ['surface', stringField(event, 'surface')],
+                    ['channel', stringField(event, 'channel')],
+                    ['pluginId', stringField(event, 'pluginId')],
+                    ['bytes', numberField(event, 'bytes')],
+                    ['limitBytes', numberField(event, 'limitBytes')],
+                    ['count', numberField(event, 'count')],
+                    ['reason', stringField(event, 'reason')],
+                ]),
+            });
+            return;
+    }
+}
+function safeTelemetryObserver(params) {
+    try {
+        params.run();
+    }
+    catch (error) {
+        params.logger.warn(`[tlon] Telemetry observer failed (${params.telemetrySource}${params.sourceEventName ? `:${params.sourceEventName}` : ''}): ${String(error)}`);
+        try {
+            reportTelemetryError({
+                telemetrySource: params.telemetrySource,
+                sourceEventName: params.sourceEventName,
+                sessionKey: params.sessionKey,
+                sessionId: params.sessionId,
+                runId: params.runId,
+                agentId: params.agentId,
+                errorKind: error instanceof Error ? error.name : typeof error,
+                errorText: formatTlonTelemetryErrorText(error),
+            });
+        }
+        catch (reportError) {
+            params.logger.warn(`[tlon] Telemetry error reporting failed: ${String(reportError)}`);
+        }
+    }
+}
+function installTelemetryDiagnosticObservers(api) {
+    return installTlonDiagnosticSubscriptions(() => {
+        const unsubscribeDiagnosticEvents = onDiagnosticEvent((event) => {
+            const candidate = event;
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'diagnostic_session',
+                sourceEventName: candidate.type,
+                sessionKey: candidate.sessionKey,
+                sessionId: candidate.sessionId,
+                run: () => {
+                    if (isTlonSessionDiagnosticEvent(candidate)) {
+                        reportSessionDiagnostic(candidate);
+                    }
+                },
+            });
+        });
+        const unsubscribeInternalDiagnosticEvents = onInternalDiagnosticEvent((event) => {
+            const candidate = event;
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'diagnostic_internal',
+                sourceEventName: stringField(candidate, 'type'),
+                sessionKey: stringField(candidate, 'sessionKey'),
+                sessionId: stringField(candidate, 'sessionId'),
+                runId: stringField(candidate, 'runId'),
+                agentId: stringField(candidate, 'agentId'),
+                run: () => reportHarnessDiagnostic(candidate),
+            });
+        });
+        return () => {
+            unsubscribeDiagnosticEvents();
+            unsubscribeInternalDiagnosticEvents();
+        };
+    });
+}
 export default defineChannelPluginEntry({
     id: 'tlon',
     name: 'Tlon',
@@ -178,7 +561,14 @@ export default defineChannelPluginEntry({
             const gsManager = createGatewayStatusManager({
                 logger: {
                     log: (m) => api.logger.info(m),
-                    error: (m) => api.logger.warn(m),
+                    error: (m) => {
+                        reportPluginError({
+                            pluginErrorSource: 'gateway_status_heartbeat',
+                            errorKind: 'heartbeat',
+                            errorText: m,
+                        });
+                        api.logger.warn(m);
+                    },
                 },
             });
             setGatewayStatusManager(gsManager);
@@ -221,21 +611,49 @@ export default defineChannelPluginEntry({
             });
         }
         // else: zero accounts configured — nothing to do
+        // Resolve the tlon tool binary once. The tool itself and version
+        // diagnostics share this path so telemetry reports what OpenClaw will
+        // actually execute.
+        const tlonBinary = resolveTlonBinary({
+            moduleDir: __dirname,
+            resolveModule: require.resolve,
+            log: (msg) => api.logger.debug?.(msg),
+        });
+        api.logger.info(`[tlon] Registering tlon tool, binary: ${tlonBinary}`);
+        setTlonSkillVersionResolver(() => readTlonSkillVersion(tlonBinary));
+        const renderTlonVersion = async () => ({
+            text: formatTlonVersionIdentity({
+                tlonSkillVersion: await resolveTlonSkillVersion(),
+            }),
+        });
+        void resolveTlonSkillVersion().then((version) => {
+            api.logger.info(`[tlon] Tlon skill version: ${version}`);
+        });
         // Register /tlon-version command
         api.registerCommand({
             name: 'tlon-version',
             description: 'Show Tlon plugin version.',
             handler: async () => {
-                return { text: `Tlon plugin v${PLUGIN_VERSION} (${PLUGIN_COMMIT})` };
+                return renderTlonVersion();
             },
         });
-        // Register the tlon tool
-        const tlonBinary = resolveTlonBinary({
-            moduleDir: __dirname,
-            resolveModule: require.resolve,
-            log: (msg) => api.logger.debug?.(msg),
+        api.registerCommand({
+            name: 'tlon',
+            description: 'Tlon plugin diagnostics. Usage: /tlon version',
+            acceptsArgs: true,
+            handler: async (ctx) => {
+                const args = (ctx.args ?? '').trim().toLowerCase();
+                if (args !== 'version') {
+                    return { text: 'Usage: /tlon version' };
+                }
+                const result = resolveBridgeForCommand(ctx);
+                if ('error' in result) {
+                    return { text: result.error };
+                }
+                return renderTlonVersion();
+            },
         });
-        api.logger.info(`[tlon] Registering tlon tool, binary: ${tlonBinary}`);
+        // Register the tlon tool
         // Capture credentials from config at registration time
         const account = resolveTlonAccount(api.config);
         const credentials = account.configured && account.url && account.ship && account.code
@@ -356,11 +774,135 @@ export default defineChannelPluginEntry({
                     },
                 }));
             }
-            recordToolCall({
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'after_tool_call',
+                sourceEventName: event.toolName,
+                sessionKey: ctx.sessionKey,
+                run: () => {
+                    recordToolCall({
+                        sessionKey: ctx.sessionKey,
+                        toolName: event.toolName,
+                        durationMs: event.durationMs,
+                        error: event.error,
+                    });
+                },
+            });
+        });
+        // ── Session lifecycle / watchdog telemetry ─────────────────────────
+        // These hooks are global to OpenClaw, so telemetry.ts filters them through
+        // session keys remembered from Tlon inbound replies before emitting.
+        api.on('session_start', (event, ctx) => {
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'session_start',
+                sourceEventName: 'session_start',
+                sessionKey: event.sessionKey ?? ctx.sessionKey,
+                sessionId: event.sessionId ?? ctx.sessionId,
+                agentId: ctx.agentId,
+                run: () => {
+                    reportSessionLifecycle({
+                        lifecycleEvent: 'session_start',
+                        sessionKey: event.sessionKey ?? ctx.sessionKey,
+                        sessionId: event.sessionId ?? ctx.sessionId,
+                        agentId: ctx.agentId,
+                        hasNextSession: false,
+                    });
+                },
+            });
+        });
+        api.on('session_end', (event, ctx) => {
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'session_end',
+                sourceEventName: 'session_end',
+                sessionKey: event.sessionKey ?? ctx.sessionKey,
+                sessionId: event.sessionId ?? ctx.sessionId,
+                agentId: ctx.agentId,
+                run: () => {
+                    reportSessionLifecycle({
+                        lifecycleEvent: 'session_end',
+                        sessionKey: event.sessionKey ?? ctx.sessionKey,
+                        sessionId: event.sessionId ?? ctx.sessionId,
+                        agentId: ctx.agentId,
+                        reason: event.reason ?? null,
+                        messageCount: event.messageCount,
+                        durationMs: event.durationMs ?? null,
+                        transcriptArchived: event.transcriptArchived ?? null,
+                        hasNextSession: Boolean(event.nextSessionId ?? event.nextSessionKey),
+                    });
+                },
+            });
+        });
+        if (shouldInstallTlonDiagnosticSubscriptions(api.registrationMode)) {
+            const unsubscribeDiagnosticEvents = installTelemetryDiagnosticObservers(api);
+            api.on('gateway_stop', unsubscribeDiagnosticEvents);
+        }
+        // ── Route diagnostics ───────────────────────────────────────────────
+        // Fires for every outbound send OpenClaw routes — the primary streamed
+        // reply (resolves to `tlon`) and route-dependent sends (the shared
+        // `message` tool, subagents, which can resolve elsewhere). `ctx.channelId`
+        // is where the send resolved; `routedToTlon: false` (e.g. `webchat`) is the
+        // leak this work targets. Read-only; never alters delivery.
+        //
+        // Two sinks: a PostHog event (the primary, fleet-wide signal — gated by the
+        // existing telemetry config, on in hosted prod) so we can count how often
+        // sends land off-Tlon; and a debug-gated local log for single-gateway
+        // triage.
+        api.on('message_sending', (event, ctx) => {
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'message_sending',
+                sourceEventName: 'message_sending',
                 sessionKey: ctx.sessionKey,
-                toolName: event.toolName,
-                durationMs: event.durationMs,
-                error: event.error,
+                runId: ctx.runId,
+                run: () => {
+                    const resolvedChannel = ctx.channelId;
+                    const routedToTlon = resolvedChannel === 'tlon';
+                    // Only infer target kind for Tlon targets; a webchat target id is not
+                    // a Tlon target and must not be misclassified.
+                    const parsedTarget = routedToTlon ? parseTlonTarget(event.to) : null;
+                    const targetKind = parsedTarget?.kind === 'dm'
+                        ? 'dm'
+                        : parsedTarget?.kind === 'channel'
+                            ? 'group'
+                            : 'unknown';
+                    reportOutboundRoute({ resolvedChannel, routedToTlon, targetKind });
+                    if (isRouteDebugEnabled()) {
+                        api.logger.info(`[tlon][route-debug] message_sending ${JSON.stringify({
+                            channelId: ctx.channelId,
+                            to: event.to,
+                            routedToTlon,
+                            targetKind,
+                            sessionKey: ctx.sessionKey ?? null,
+                            conversationId: ctx.conversationId ?? null,
+                            messageId: ctx.messageId ?? null,
+                            threadId: event.threadId ?? null,
+                        })}`);
+                    }
+                },
+            });
+        });
+        api.on('message_sent', (event, ctx) => {
+            safeTelemetryObserver({
+                logger: api.logger,
+                telemetrySource: 'message_sent',
+                sourceEventName: 'message_sent',
+                sessionKey: event.sessionKey ?? ctx.sessionKey,
+                runId: event.runId ?? ctx.runId,
+                run: () => {
+                    if (event.success !== false) {
+                        return;
+                    }
+                    reportHarnessError({
+                        harnessEventType: 'message_sent',
+                        errorScope: 'message_delivery',
+                        sessionKey: event.sessionKey ?? ctx.sessionKey,
+                        runId: event.runId ?? ctx.runId,
+                        errorText: event.error ?? null,
+                        outcome: 'error',
+                    });
+                },
             });
         });
         // ── Slash commands for approval & admin ────────────────────────────