@2en/clawly-plugins 1.25.0 → 1.26.0-beta.1

package/gateway/cron-telemetry.ts

@@ -0,0 +1,253 @@
+/**
+ * Cron lifecycle telemetry — captures creation, execution, and deletion
+ * events for cron jobs via the plugin hook system.
+ *
+ * Events are emitted as OTel LogRecords via @opentelemetry/sdk-logs when
+ * plugin telemetry config contains Axiom credentials.
+ * Human-readable logs are always emitted via api.logger regardless.
+ *
+ * Hooks:
+ * - after_tool_call — captures cron add/remove tool results
+ * - agent_end (priority 100) — captures cron execution outcomes
+ *   Runs after cron-delivery and offline-push so delivery/push flags are set.
+ *
+ * Cross-hook coordination:
+ * - markCronDelivered(sessionKey) — called by cron-delivery on successful inject
+ * - markCronDeliverySkipped(sessionKey, reason) — called by cron-delivery on skip
+ * - markCronPushSent(sessionKey) — called by offline-push on successful push
+ * - markCronPushSkipped(sessionKey, reason, clientOnline) — called by offline-push on skip
+ */
+
+import {SeverityNumber} from '@opentelemetry/api-logs'
+import type {PluginApi} from '../types'
+import {getOtelLogger} from './otel'
+import {captureEvent} from './posthog'
+import {readTelemetryPluginConfig} from './telemetry-config'
+
+// ── Structured telemetry attributes (OTel LogRecord) ────────────
+
+export type CronTelemetryAttributes = {
+  'cron.lifecycle': 'created' | 'executed' | 'deleted'
+  'cron.job.id': string
+  'cron.job.name'?: string
+  'cron.job.schedule'?: string
+  'cron.execution.success'?: boolean
+  'cron.execution.error'?: string
+  'cron.execution.duration_ms'?: number
+  'cron.delivery.app_message_sent'?: boolean
+  'cron.delivery.app_message_skip_reason'?: string
+  'cron.delivery.push_notification_sent'?: boolean
+  'cron.delivery.push_skip_reason'?: string
+  'instance.id'?: string
+  'agent.id'?: string
+  'client.online'?: boolean
+}
+
+// ── Cross-hook coordination flags ───────────────────────────────
+
+type CronRunFlags = {
+  delivered: boolean
+  deliverySkipReason?: string
+  pushSent: boolean
+  pushSkipReason?: string
+  clientOnline?: boolean
+  createdAt: number
+}
+
+const cronRunFlags = new Map<string, CronRunFlags>()
+
+/** Max age for flag entries (10 minutes). Safety net against leaks. */
+const FLAG_TTL_MS = 10 * 60 * 1000
+
+function getOrCreate(sessionKey: string): CronRunFlags {
+  let flags = cronRunFlags.get(sessionKey)
+  if (!flags) {
+    // Evict stale entries on insertion to bound map size
+    if (cronRunFlags.size >= 50) {
+      const now = Date.now()
+      for (const [key, val] of cronRunFlags) {
+        if (now - val.createdAt > FLAG_TTL_MS) cronRunFlags.delete(key)
+      }
+    }
+    flags = {delivered: false, pushSent: false, createdAt: Date.now()}
+    cronRunFlags.set(sessionKey, flags)
+  }
+  return flags
+}
+
+export function markCronDelivered(sessionKey: string) {
+  getOrCreate(sessionKey).delivered = true
+}
+
+export function markCronDeliverySkipped(sessionKey: string, reason: string) {
+  const flags = getOrCreate(sessionKey)
+  flags.delivered = false
+  flags.deliverySkipReason = reason
+}
+
+export function markCronPushSent(sessionKey: string) {
+  getOrCreate(sessionKey).pushSent = true
+}
+
+export function markCronPushSkipped(sessionKey: string, reason: string, clientOnline?: boolean) {
+  const flags = getOrCreate(sessionKey)
+  flags.pushSent = false
+  flags.pushSkipReason = reason
+  if (clientOnline != null) flags.clientOnline = clientOnline
+}
+
+// ── Helpers ──────────────────────────────────────────────────────
+
+function extractJobId(sessionKey: string): string | null {
+  const prefix = 'agent:clawly:cron:'
+  if (!sessionKey.startsWith(prefix)) return null
+  return sessionKey.slice(prefix.length) || null
+}
+
+function emit(attributes: CronTelemetryAttributes, api: PluginApi) {
+  // Emit structured OTel LogRecord (if provider is initialized)
+  const otelLogger = getOtelLogger()
+  if (otelLogger) {
+    otelLogger.emit({
+      severityNumber: SeverityNumber.INFO,
+      severityText: 'INFO',
+      body: `cron.lifecycle.${attributes['cron.lifecycle']}`,
+      attributes: attributes as Record<string, unknown>,
+    })
+  }
+
+  // Emit PostHog event
+  captureEvent(`cron.${attributes['cron.lifecycle']}`, attributes as Record<string, unknown>)
+
+  // Always log human-readable summary
+  const a = attributes
+  switch (a['cron.lifecycle']) {
+    case 'created':
+      api.logger.info(
+        `cron-telemetry: job created jobId=${a['cron.job.id']}${a['cron.job.name'] ? ` name="${a['cron.job.name']}"` : ''}${a['cron.job.schedule'] ? ` schedule="${a['cron.job.schedule']}"` : ''}`,
+      )
+      break
+    case 'deleted':
+      api.logger.info(`cron-telemetry: job deleted jobId=${a['cron.job.id']}`)
+      break
+    case 'executed':
+      api.logger.info(
+        `cron-telemetry: job executed jobId=${a['cron.job.id']} success=${a['cron.execution.success']} appMessage=${a['cron.delivery.app_message_sent']} push=${a['cron.delivery.push_notification_sent']}${a['cron.execution.duration_ms'] != null ? ` duration=${a['cron.execution.duration_ms']}ms` : ''}`,
+      )
+      break
+  }
+}
+
+// ── Registration ─────────────────────────────────────────────────
+
+export function registerCronTelemetry(api: PluginApi) {
+  // 1. after_tool_call: capture cron add/remove
+  api.on('after_tool_call', (event: Record<string, unknown>, ctx?: Record<string, unknown>) => {
+    api.logger.info(
+      `cron-telemetry[debug]: after_tool_call fired toolName=${String(event.toolName)} params.action=${String((event.params as any)?.action)} ctx=${JSON.stringify(ctx ?? {})}`,
+    )
+    if (event.toolName !== 'cron') return
+
+    const params = (event.params ?? {}) as Record<string, unknown>
+    const result = event.result as Record<string, unknown> | undefined
+    api.logger.info(
+      `cron-telemetry[debug]: cron tool detected action=${String(params.action)} result=${JSON.stringify(result ?? null)} error=${String(event.error ?? 'none')}`,
+    )
+
+    if (params.action === 'add' && result) {
+      const job = (params.job ?? params) as Record<string, unknown>
+      const jobId =
+        typeof result.id === 'string'
+          ? result.id
+          : typeof result.jobId === 'string'
+            ? result.jobId
+            : 'unknown'
+      const jobName = typeof job.name === 'string' ? job.name : undefined
+      const schedule = typeof job.schedule === 'string' ? job.schedule : undefined
+
+      emit(
+        {
+          'cron.lifecycle': 'created',
+          'cron.job.id': jobId,
+          ...(jobName ? {'cron.job.name': jobName} : {}),
+          ...(schedule ? {'cron.job.schedule': schedule} : {}),
+        },
+        api,
+      )
+    }
+
+    if (params.action === 'remove') {
+      const jobId =
+        typeof params.id === 'string'
+          ? params.id
+          : typeof params.jobId === 'string'
+            ? params.jobId
+            : 'unknown'
+
+      emit({'cron.lifecycle': 'deleted', 'cron.job.id': jobId}, api)
+    }
+  })
+
+  // Read instance.id once at registration time
+  const instanceId =
+    readTelemetryPluginConfig(api.pluginConfig).instanceId ?? process.env.INSTANCE_ID
+
+  // 2. agent_end: capture cron execution (priority 100 = runs after delivery/push hooks)
+  api.on(
+    'agent_end',
+    (event: Record<string, unknown>, ctx?: Record<string, unknown>) => {
+      const sessionKey = typeof ctx?.sessionKey === 'string' ? ctx.sessionKey : undefined
+      api.logger.info(
+        `cron-telemetry[debug]: agent_end fired trigger=${String(ctx?.trigger ?? 'undefined')} sessionKey=${sessionKey ?? 'undefined'} agentId=${String(ctx?.agentId ?? 'undefined')} eventKeys=${Object.keys(event).join(',')}`,
+      )
+
+      // Detect cron sessions by sessionKey prefix (consistent with cron-delivery)
+      // instead of ctx.trigger which may not be set by all OpenClaw versions.
+      if (!sessionKey?.startsWith('agent:clawly:cron:')) return
+
+      const jobId = extractJobId(sessionKey)
+      if (!jobId) {
+        api.logger.info(
+          `cron-telemetry[debug]: agent_end skipped — extractJobId returned null for sessionKey=${sessionKey}`,
+        )
+        return
+      }
+
+      const agentId = typeof ctx.agentId === 'string' ? ctx.agentId : undefined
+      const flags: CronRunFlags = cronRunFlags.get(sessionKey) ?? {
+        delivered: false,
+        pushSent: false,
+      }
+      cronRunFlags.delete(sessionKey)
+
+      const success = event.error == null
+      const error = typeof event.error === 'string' ? event.error : undefined
+      const durationMs = typeof event.durationMs === 'number' ? event.durationMs : undefined
+
+      emit(
+        {
+          'cron.lifecycle': 'executed',
+          'cron.job.id': jobId,
+          'cron.execution.success': success,
+          ...(error ? {'cron.execution.error': error} : {}),
+          ...(durationMs != null ? {'cron.execution.duration_ms': durationMs} : {}),
+          'cron.delivery.app_message_sent': flags.delivered,
+          ...(flags.deliverySkipReason
+            ? {'cron.delivery.app_message_skip_reason': flags.deliverySkipReason}
+            : {}),
+          'cron.delivery.push_notification_sent': flags.pushSent,
+          ...(flags.pushSkipReason ? {'cron.delivery.push_skip_reason': flags.pushSkipReason} : {}),
+          ...(instanceId ? {'instance.id': instanceId} : {}),
+          ...(agentId ? {'agent.id': agentId} : {}),
+          ...(flags.clientOnline != null ? {'client.online': flags.clientOnline} : {}),
+        },
+        api,
+      )
+    },
+    {priority: 100},
+  )
+
+  api.logger.info(
+    'cron-telemetry: registered after_tool_call + agent_end hooks (debug-instrumented)',
+  )
+}

package/gateway/index.ts

@@ -1,8 +1,12 @@
 import type {PluginApi} from '../types'
 import {registerAgentSend} from './agent'
+import {registerAnalytics} from './analytics'
 import {registerClawhub2gateway} from './clawhub2gateway'
 import {registerConfigRepair} from './config-repair'
 import {registerCronDelivery} from './cron-delivery'
+import {registerCronTelemetry} from './cron-telemetry'
+import {initOtel, shutdownOtel} from './otel'
+import {initPostHog, setOpenClawVersion, shutdownPostHog} from './posthog'
 import {registerPairing} from './pairing'
 import {registerMemoryBrowser} from './memory'
 import {registerNotification} from './notification'
@@ -12,6 +16,33 @@ import {registerPresence} from './presence'
 import {registerVersion} from './version'
 
 export function registerGateway(api: PluginApi) {
+  // Initialize OTel logs provider (no-op if plugin telemetry config is missing)
+  const otelEnabled = initOtel(api.pluginConfig)
+  if (otelEnabled) {
+    api.logger.info('otel: LoggerProvider initialized (OTLP/HTTP)')
+  }
+
+  // Initialize PostHog analytics (no-op if plugin telemetry config is missing)
+  const posthogEnabled = initPostHog(api.pluginConfig)
+  if (posthogEnabled) {
+    const openclawVersion = api.config?.meta?.lastTouchedVersion
+    if (openclawVersion) setOpenClawVersion(openclawVersion)
+    api.logger.info('posthog: client initialized')
+  }
+
+  if (otelEnabled || posthogEnabled) {
+    api.on('gateway_stop', async () => {
+      if (otelEnabled) {
+        await shutdownOtel()
+        api.logger.info('otel: LoggerProvider shut down')
+      }
+      if (posthogEnabled) {
+        await shutdownPostHog()
+        api.logger.info('posthog: client shut down')
+      }
+    })
+  }
+
   registerPresence(api)
   registerNotification(api)
   registerAgentSend(api)
@@ -20,6 +51,8 @@ export function registerGateway(api: PluginApi) {
   registerPlugins(api)
   registerOfflinePush(api)
   registerCronDelivery(api)
+  registerCronTelemetry(api)
+  registerAnalytics(api)
   registerConfigRepair(api)
   registerPairing(api)
   registerVersion(api)

package/gateway/offline-push.test.ts

@@ -3,6 +3,8 @@ import type {PluginApi} from '../types'
 import {
   getLastAssistantPreview,
   getLastAssistantText,
+  getTriggeringUserText,
+  isHeartbeatTriggered,
   registerOfflinePush,
   shouldSkipPushForMessage,
 } from './offline-push'
@@ -515,3 +517,210 @@ describe('offline-push with filtered messages', () => {
     })
   })
 })
+
+// ── getTriggeringUserText unit tests ─────────────────────────────
+
+describe('getTriggeringUserText', () => {
+  test('returns null for non-array input', () => {
+    expect(getTriggeringUserText(undefined)).toBeNull()
+    expect(getTriggeringUserText(null)).toBeNull()
+    expect(getTriggeringUserText('string')).toBeNull()
+  })
+
+  test('extracts user text preceding the last assistant message', () => {
+    const messages = [
+      {role: 'user', content: 'Hello'},
+      {role: 'assistant', content: 'Hi there!'},
+    ]
+    expect(getTriggeringUserText(messages)).toBe('Hello')
+  })
+
+  test('returns the user message just before the last assistant, not earlier ones', () => {
+    const messages = [
+      {role: 'user', content: 'First question'},
+      {role: 'assistant', content: 'First reply'},
+      {role: 'user', content: 'Second question'},
+      {role: 'assistant', content: 'Second reply'},
+    ]
+    expect(getTriggeringUserText(messages)).toBe('Second question')
+  })
+
+  test('skips system messages between assistant and user', () => {
+    const messages = [
+      {role: 'user', content: 'Trigger text'},
+      {role: 'system', content: 'System instruction'},
+      {role: 'assistant', content: 'Response'},
+    ]
+    expect(getTriggeringUserText(messages)).toBe('Trigger text')
+  })
+
+  test('returns null when no user message precedes assistant', () => {
+    const messages = [{role: 'assistant', content: 'Unprompted response'}]
+    expect(getTriggeringUserText(messages)).toBeNull()
+  })
+
+  test('returns null when no assistant message exists', () => {
+    const messages = [{role: 'user', content: 'Hello'}]
+    expect(getTriggeringUserText(messages)).toBeNull()
+  })
+
+  test('handles array content format for user message', () => {
+    const messages = [
+      {
+        role: 'user',
+        content: [
+          {type: 'text', text: 'Read HEARTBEAT.md'},
+          {type: 'text', text: ' and follow instructions'},
+        ],
+      },
+      {role: 'assistant', content: 'HEARTBEAT_OK'},
+    ]
+    expect(getTriggeringUserText(messages)).toBe('Read HEARTBEAT.md and follow instructions')
+  })
+
+  test('returns null for empty messages array', () => {
+    expect(getTriggeringUserText([])).toBeNull()
+  })
+})
+
+// ── isHeartbeatTriggered unit tests ──────────────────────────────
+
+describe('isHeartbeatTriggered', () => {
+  test('returns true for heartbeat prompt', () => {
+    const messages = [
+      {role: 'user', content: 'Read HEARTBEAT.md and follow the instructions inside.'},
+      {role: 'assistant', content: 'HEARTBEAT_OK'},
+    ]
+    expect(isHeartbeatTriggered(messages)).toBe(true)
+  })
+
+  test('returns true for verbose heartbeat response', () => {
+    const messages = [
+      {role: 'user', content: 'Read HEARTBEAT.md and follow the instructions inside.'},
+      {
+        role: 'assistant',
+        content: 'Checked your inbox, nothing new. All clear! HEARTBEAT_OK',
+      },
+    ]
+    expect(isHeartbeatTriggered(messages)).toBe(true)
+  })
+
+  test('returns true for heartbeat response without HEARTBEAT_OK', () => {
+    const messages = [
+      {role: 'user', content: 'Read HEARTBEAT.md and follow the instructions inside.'},
+      {
+        role: 'assistant',
+        content: 'Nothing to report right now. Everything looks quiet.',
+      },
+    ]
+    expect(isHeartbeatTriggered(messages)).toBe(true)
+  })
+
+  test('returns false for normal user messages', () => {
+    const messages = [
+      {role: 'user', content: 'What is the weather today?'},
+      {role: 'assistant', content: 'Let me check...'},
+    ]
+    expect(isHeartbeatTriggered(messages)).toBe(false)
+  })
+
+  test('returns false for no messages', () => {
+    expect(isHeartbeatTriggered([])).toBe(false)
+    expect(isHeartbeatTriggered(undefined)).toBe(false)
+  })
+})
+
+// ── Heartbeat-triggered integration tests ────────────────────────
+
+describe('offline-push with heartbeat-triggered turns', () => {
+  test('skips push for verbose heartbeat response', async () => {
+    const {api, logs, handlers} = createMockApi()
+    registerOfflinePush(api)
+
+    await handlers.get('agent_end')!(
+      {
+        messages: [
+          {role: 'user', content: 'Read HEARTBEAT.md and follow the instructions inside.'},
+          {
+            role: 'assistant',
+            content: 'Checked your inbox, nothing new. All clear! HEARTBEAT_OK',
+          },
+        ],
+      },
+      {sessionKey: 'agent:clawly:main'},
+    )
+
+    expect(logs).toContainEqual({
+      level: 'info',
+      msg: 'offline-push: skipped (heartbeat-triggered turn)',
+    })
+    expect(logs.filter((l) => l.msg.includes('notified'))).toHaveLength(0)
+  })
+
+  test('skips push for heartbeat response without HEARTBEAT_OK sentinel', async () => {
+    const {api, logs, handlers} = createMockApi()
+    registerOfflinePush(api)
+
+    await handlers.get('agent_end')!(
+      {
+        messages: [
+          {role: 'user', content: 'Read HEARTBEAT.md and follow the instructions inside.'},
+          {
+            role: 'assistant',
+            content: 'Nothing to report. Everything looks quiet on all fronts.',
+          },
+        ],
+      },
+      {sessionKey: 'agent:clawly:main'},
+    )
+
+    expect(logs).toContainEqual({
+      level: 'info',
+      msg: 'offline-push: skipped (heartbeat-triggered turn)',
+    })
+    expect(logs.filter((l) => l.msg.includes('notified'))).toHaveLength(0)
+  })
+
+  test('bare HEARTBEAT_OK is caught by existing filter before heartbeat-context check', async () => {
+    const {api, logs, handlers} = createMockApi()
+    registerOfflinePush(api)
+
+    await handlers.get('agent_end')!(
+      {
+        messages: [
+          {role: 'user', content: 'Read HEARTBEAT.md and follow the instructions inside.'},
+          {role: 'assistant', content: 'HEARTBEAT_OK'},
+        ],
+      },
+      {sessionKey: 'agent:clawly:main'},
+    )
+
+    // Should be caught by shouldSkipPushForMessage (heartbeat ack), not the trigger check
+    expect(logs).toContainEqual({
+      level: 'info',
+      msg: expect.stringContaining('skipped (filtered: heartbeat ack)'),
+    })
+    expect(logs.filter((l) => l.msg.includes('heartbeat-triggered turn'))).toHaveLength(0)
+  })
+
+  test('non-heartbeat turn sends push normally (no regression)', async () => {
+    const {api, logs, handlers} = createMockApi()
+    registerOfflinePush(api)
+
+    await handlers.get('agent_end')!(
+      {
+        messages: [
+          {role: 'user', content: 'What is the weather like today?'},
+          {role: 'assistant', content: 'It is sunny and 72°F!'},
+        ],
+      },
+      {sessionKey: 'agent:clawly:main'},
+    )
+
+    expect(logs).toContainEqual({
+      level: 'info',
+      msg: expect.stringContaining('notified (session=agent:clawly:main)'),
+    })
+    expect(logs.filter((l) => l.msg.includes('heartbeat-triggered'))).toHaveLength(0)
+  })
+})