@controlflow-ai/daemon 0.1.2 → 0.1.3

package/src/messaging-status.ts

@@ -0,0 +1,669 @@
+import type { DeliveryBacklogSummary } from './db.js';
+export type { DeliveryBacklogSummary } from './db.js';
+
+export interface DeliveryConnectionStats {
+  connection_id: string;
+  computer_id: string | null;
+  open_sockets: number;
+  last_open_at: string | null;
+  last_close_at: string | null;
+  last_ping_at: string | null;
+  last_pong_at: string | null;
+  last_close_code: number | null;
+  last_close_reason: string | null;
+  pending_agents: Array<{ agent: string; pending: number }>;
+  backlog: DeliveryBacklogSummary[];
+}
+
+export interface DeliveryWebSocketSummary {
+  connections: DeliveryConnectionStats[];
+  totals: {
+    connections: number;
+    open_sockets: number;
+    pending_deliveries: number;
+    claimed_deliveries: number;
+    processing_completed_deliveries: number;
+    expired_active_deliveries: number;
+  };
+}
+
+export interface MessagingDiagnostic {
+  code: string;
+  level: 'ok' | 'warn';
+  message: string;
+  hint?: string;
+}
+
+export interface MessagingStatusSummary {
+  level: 'ok' | 'warn';
+  codes: string[];
+  warn_count: number;
+}
+
+export interface LarkMessagingStatus {
+  app_id: string;
+  agent?: string | null;
+  label?: string | null;
+  bot_open_id_known?: boolean;
+  authorized_user_count?: number;
+  runtime_home?: string;
+  config_path?: string;
+  db_path?: string | null;
+  ws_state: string | null;
+  ws_last_connect_at?: string | null;
+  ws_next_connect_at?: string | null;
+  ws_reconnect_attempts?: number | null;
+  ws_last_event_at?: string | null;
+  ws_last_event_type?: string | null;
+  ws_last_error?: string | null;
+  restart_count?: number;
+  last_restart_at?: string | null;
+  last_restart_reason?: string | null;
+  event_count?: number;
+  message_event_count?: number;
+  provider_event_count?: number;
+  provider_message_event_count?: number;
+  last_provider_event_at?: string | null;
+  last_provider_message_event_at?: string | null;
+  db_event_count?: number;
+  db_provider_event_count?: number;
+  db_probe_event_count?: number;
+  db_parse_error_count?: number;
+  db_last_event?: unknown;
+  db_last_probe_event?: unknown;
+  last_ingest_status?: string | null;
+  last_delivery_notify?: unknown;
+  last_error?: string | null;
+}
+
+export interface MessagingStatus {
+  summary: MessagingStatusSummary;
+  diagnostics: MessagingDiagnostic[];
+  lark: { bots: LarkMessagingStatus[] };
+  delivery_websocket: DeliveryWebSocketSummary;
+}
+
+export interface MessagingHealthDomain {
+  level: 'ok' | 'warn';
+  codes: string[];
+  message: string;
+  hint?: string;
+  recommended_action?: string;
+  details?: Record<string, unknown>;
+}
+
+export interface MessagingHealth {
+  level: 'ok' | 'warn';
+  domains: {
+    lark_provider: MessagingHealthDomain;
+    delivery_websocket: MessagingHealthDomain;
+    delivery_backlog: MessagingHealthDomain;
+  };
+  summary: MessagingStatusSummary;
+}
+
+export interface MessagingHealthLogLine {
+  throttle_key: string;
+  message: string;
+}
+
+type MessagingHealthDomainName = keyof MessagingHealth['domains'];
+
+interface LarkStatusForDiagnostics {
+  app_id?: unknown;
+  ws_state?: unknown;
+  agent?: unknown;
+  bot_open_id_known?: unknown;
+  authorized_user_count?: unknown;
+  event_count?: unknown;
+  provider_event_count?: unknown;
+  db_event_count?: unknown;
+  db_provider_event_count?: unknown;
+  db_parse_error_count?: unknown;
+  db_last_event?: unknown;
+  last_error?: unknown;
+  ws_last_error?: unknown;
+  restart_count?: unknown;
+  last_restart_at?: unknown;
+  last_restart_reason?: unknown;
+}
+
+const DEFAULT_LARK_EVENT_STALE_AFTER_MS = 10 * 60 * 1000;
+const DEFAULT_DELIVERY_WS_PONG_STALE_AFTER_MS = 2 * 60 * 1000;
+const DEFAULT_LARK_RESTART_EFFECT_GRACE_MS = 60 * 1000;
+
+function numberField(value: unknown): number {
+  return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+
+function providerDbEventCount(bot: Pick<LarkMessagingStatus, 'db_event_count' | 'db_provider_event_count'> | { db_event_count?: unknown; db_provider_event_count?: unknown }): number {
+  return numberField(bot.db_provider_event_count ?? bot.db_event_count);
+}
+
+function stringField(value: unknown): string | null {
+  return typeof value === 'string' && value.trim() ? value : null;
+}
+
+function larkEventStaleAfterMs(): number {
+  const raw = process.env.PAL_LARK_EVENT_STALE_AFTER_MS;
+  if (!raw) return DEFAULT_LARK_EVENT_STALE_AFTER_MS;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_LARK_EVENT_STALE_AFTER_MS;
+}
+
+function deliveryWsPongStaleAfterMs(): number {
+  const raw = process.env.PAL_DELIVERY_WS_PONG_STALE_AFTER_MS;
+  if (!raw) return DEFAULT_DELIVERY_WS_PONG_STALE_AFTER_MS;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_DELIVERY_WS_PONG_STALE_AFTER_MS;
+}
+
+function larkRestartEffectGraceMs(): number {
+  const raw = process.env.PAL_LARK_RESTART_EFFECT_GRACE_MS;
+  if (!raw) return DEFAULT_LARK_RESTART_EFFECT_GRACE_MS;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_LARK_RESTART_EFFECT_GRACE_MS;
+}
+
+function parseTimestamp(value: unknown): number | null {
+  const text = stringField(value);
+  if (!text) return null;
+  const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(text)
+    ? `${text.replace(' ', 'T')}Z`
+    : text;
+  const time = Date.parse(normalized);
+  return Number.isFinite(time) ? time : null;
+}
+
+function parseInboundReceivedAt(value: unknown): number | null {
+  if (!value || typeof value !== 'object') return null;
+  return parseTimestamp((value as { received_at?: unknown }).received_at);
+}
+
+function inboundParseOk(value: unknown): boolean | null {
+  if (!value || typeof value !== 'object') return null;
+  const parseOk = (value as { parse_ok?: unknown }).parse_ok;
+  if (parseOk === 1 || parseOk === true) return true;
+  if (parseOk === 0 || parseOk === false) return false;
+  return null;
+}
+
+function latestEventReceivedAt(bot: LarkMessagingStatus): string | null {
+  if (!bot.db_last_event || typeof bot.db_last_event !== 'object') return null;
+  const value = (bot.db_last_event as { received_at?: unknown }).received_at;
+  return typeof value === 'string' && value.trim() ? value : null;
+}
+
+function latestProbeEventReceivedAt(bot: LarkMessagingStatus): string | null {
+  if (!bot.db_last_probe_event || typeof bot.db_last_probe_event !== 'object') return null;
+  const value = (bot.db_last_probe_event as { received_at?: unknown }).received_at;
+  return typeof value === 'string' && value.trim() ? value : null;
+}
+
+function warningDiagnostics(diagnostics: MessagingDiagnostic[], codes: string[]): MessagingDiagnostic[] {
+  const wanted = new Set(codes);
+  return diagnostics.filter((diagnostic) => diagnostic.level === 'warn' && wanted.has(diagnostic.code));
+}
+
+function prioritizedDiagnostics(diagnostics: MessagingDiagnostic[], priority: string[]): MessagingDiagnostic[] {
+  const rank = new Map(priority.map((code, index) => [code, index]));
+  return [...diagnostics].sort((a, b) => (rank.get(a.code) ?? Number.MAX_SAFE_INTEGER) - (rank.get(b.code) ?? Number.MAX_SAFE_INTEGER));
+}
+
+function firstHint(diagnostics: MessagingDiagnostic[]): string | undefined {
+  return diagnostics.find((diagnostic) => diagnostic.hint)?.hint;
+}
+
+function larkProviderHint(codes: string[], diagnostics: MessagingDiagnostic[]): string | undefined {
+  const present = new Set(codes);
+  const clusterHint = 'Feishu long-connection delivery is cluster-mode, so if another client for the same app is online, fresh events may be delivered there instead of this Pal server.';
+  if (
+    !present.has('lark_restart_ineffective')
+    && present.has('lark_no_events_since_restart')
+    && present.has('lark_events_stale')
+  ) {
+    return `Pal is connected to Lark but fresh Feishu messages are not reaching this server; inspect Feishu long-connection event subscription and other consumers of this app stream. ${clusterHint}`;
+  }
+  const hint = firstHint(diagnostics);
+  return hint && present.has('lark_restart_ineffective') ? `${hint} ${clusterHint}` : hint;
+}
+
+function staleDeliveryHeartbeatConnections(summary: DeliveryWebSocketSummary): DeliveryConnectionStats[] {
+  if (summary.totals.open_sockets <= 0) return [];
+  const now = Date.now();
+  const staleAfterMs = deliveryWsPongStaleAfterMs();
+  return summary.connections.filter((connection) => {
+    if (connection.open_sockets <= 0) return false;
+    const lastPongAt = parseTimestamp(connection.last_pong_at);
+    if (lastPongAt !== null) return now - lastPongAt > staleAfterMs;
+    const referenceAt = parseTimestamp(connection.last_ping_at) ?? parseTimestamp(connection.last_open_at);
+    return referenceAt !== null && now - referenceAt > staleAfterMs;
+  });
+}
+
+function recommendedAction(codes: string[], rules: Array<[string, string]>): string | undefined {
+  const present = new Set(codes);
+  return rules.find(([code]) => present.has(code))?.[1];
+}
+
+function recommendLarkStaleNoIngressAction(codes: string[]): string | undefined {
+  const present = new Set(codes);
+  if (present.has('lark_no_events_since_restart') && present.has('lark_events_stale')) {
+    return 'inspect_feishu_event_subscription';
+  }
+  return undefined;
+}
+
+const LARK_PROVIDER_ACTION_RULES: Array<[string, string]> = [
+  ['lark_error', 'repair_lark_websocket'],
+  ['lark_ws_not_connected', 'repair_lark_websocket'],
+  ['lark_restart_ineffective', 'inspect_feishu_event_subscription'],
+  ['lark_probe_only_since_restart', 'inspect_feishu_event_subscription'],
+  ['lark_no_events_since_restart', 'send_real_feishu_message_then_watch_lark'],
+  ['lark_events_stale', 'repair_lark_websocket'],
+  ['lark_no_bots', 'configure_lark_bot'],
+  ['lark_no_bound_agent', 'bind_lark_bot_agent'],
+  ['lark_no_authorized_users', 'authorize_lark_user'],
+  ['lark_bot_open_id_unknown', 'refresh_lark_binding'],
+  ['lark_raw_event_parse_errors', 'repair_lark_event_parse'],
+];
+
+export function recommendLarkProviderAction(codes: string[]): string | undefined {
+  return recommendLarkStaleNoIngressAction(codes) ?? recommendedAction(codes, LARK_PROVIDER_ACTION_RULES);
+}
+
+export function emptyDeliveryWebSocketSummary(): DeliveryWebSocketSummary {
+  return {
+    connections: [],
+    totals: {
+      connections: 0,
+      open_sockets: 0,
+      pending_deliveries: 0,
+      claimed_deliveries: 0,
+      processing_completed_deliveries: 0,
+      expired_active_deliveries: 0,
+    },
+  };
+}
+
+export function buildMessagingStatus(input: { larkBots: LarkMessagingStatus[]; deliveryWebSocket: DeliveryWebSocketSummary }): MessagingStatus {
+  const diagnostics = buildMessagingDiagnostics({
+    larkBots: input.larkBots,
+    deliveryWebSocket: input.deliveryWebSocket,
+  });
+  return {
+    summary: summarizeMessagingDiagnostics(diagnostics),
+    diagnostics,
+    lark: { bots: input.larkBots },
+    delivery_websocket: input.deliveryWebSocket,
+  };
+}
+
+export function buildMessagingDiagnostics(input: { larkBots: unknown[]; deliveryWebSocket: DeliveryWebSocketSummary }): MessagingDiagnostic[] {
+  const diagnostics: MessagingDiagnostic[] = [];
+  const staleAfterMs = larkEventStaleAfterMs();
+  const now = Date.now();
+  if (input.larkBots.length === 0) {
+    diagnostics.push({
+      code: 'lark_no_bots',
+      level: 'warn',
+      message: 'No Lark bots are active on this server.',
+      hint: 'Configure a Lark bot and reload the server integration before expecting Feishu messages.',
+    });
+  }
+  for (const bot of input.larkBots as LarkStatusForDiagnostics[]) {
+    const appId = stringField(bot.app_id) ?? 'unknown';
+    const wsState = stringField(bot.ws_state);
+    const boundAgent = stringField(bot.agent);
+    const hasAgentField = Object.prototype.hasOwnProperty.call(bot, 'agent');
+    const botOpenIdKnown = bot.bot_open_id_known === true;
+    const hasBotOpenIdKnownField = typeof bot.bot_open_id_known === 'boolean';
+    const hasAuthorizedUserCountField = typeof bot.authorized_user_count === 'number';
+    const authorizedUserCount = numberField(bot.authorized_user_count);
+    const totalEventCount = numberField(bot.event_count);
+    const hasProviderEventCountField = typeof bot.provider_event_count === 'number';
+    const eventCount = numberField(hasProviderEventCountField ? bot.provider_event_count : bot.event_count);
+    const dbEventCount = providerDbEventCount(bot);
+    const dbParseErrorCount = numberField(bot.db_parse_error_count);
+    const hasDbLastEvent = Boolean(bot.db_last_event);
+    const dbLastEventAt = parseInboundReceivedAt(bot.db_last_event);
+    const latestParseOk = inboundParseOk(bot.db_last_event);
+    const latestEventIsStale = dbLastEventAt !== null && now - dbLastEventAt > staleAfterMs;
+    const lastError = stringField(bot.last_error) ?? stringField(bot.ws_last_error);
+    const restartCount = numberField(bot.restart_count);
+    const lastRestartAt = parseTimestamp(bot.last_restart_at);
+    const lastRestartReason = stringField(bot.last_restart_reason);
+    const restartGraceMs = larkRestartEffectGraceMs();
+    if (lastError) {
+      diagnostics.push({
+        code: 'lark_error',
+        level: 'warn',
+        message: `Lark bot ${appId} reports error: ${lastError}`,
+        hint: 'Restart this Lark websocket after fixing the reported error.',
+      });
+    } else if (wsState !== 'connected') {
+      diagnostics.push({
+        code: 'lark_ws_not_connected',
+        level: 'warn',
+        message: `Lark bot ${appId} websocket state is ${wsState ?? 'unknown'}.`,
+        hint: 'Run messaging repair-lark; if it cannot reconnect, check the app credentials and network path.',
+      });
+    } else if (eventCount === 0 && dbEventCount > 0 && hasDbLastEvent) {
+      diagnostics.push({
+        code: 'lark_no_events_since_restart',
+        level: 'warn',
+        message: `Lark bot ${appId} is connected, but this server process has not received provider events since restart.`,
+        hint: 'Send a fresh Feishu message; if this stays at zero, inspect Feishu event subscription/long-connection configuration and stop other clients for the same app stream.',
+      });
+    } else {
+      diagnostics.push({ code: 'lark_ws_connected', level: 'ok', message: `Lark bot ${appId} websocket is connected.` });
+    }
+    if (hasProviderEventCountField && eventCount === 0 && totalEventCount > 0) {
+      diagnostics.push({
+        code: 'lark_probe_only_since_restart',
+        level: 'warn',
+        message: `Lark bot ${appId} has only received local probe events since restart, not Feishu provider events.`,
+        hint: 'Pal local ingestion and delivery probing can pass while Feishu provider delivery is still broken; use watch-lark with a real Feishu message.',
+      });
+    }
+    if (hasAgentField && !boundAgent) {
+      diagnostics.push({
+        code: 'lark_no_bound_agent',
+        level: 'warn',
+        message: `Lark bot ${appId} is not bound to an agent.`,
+        hint: 'Bind this Lark bot to an agent before expecting inbound Feishu messages to create agent work.',
+      });
+    }
+    if (hasAuthorizedUserCountField && authorizedUserCount === 0) {
+      diagnostics.push({
+        code: 'lark_no_authorized_users',
+        level: 'warn',
+        message: `Lark bot ${appId} has no authorized sender union IDs configured.`,
+        hint: 'Add at least one authorized Feishu user with lark-users add; otherwise inbound messages are stored but skipped.',
+      });
+    }
+    if (hasBotOpenIdKnownField && !botOpenIdKnown) {
+      diagnostics.push({
+        code: 'lark_bot_open_id_unknown',
+        level: 'warn',
+        message: `Lark bot ${appId} does not have a known bot open_id in local config.`,
+        hint: 'Refresh the Lark binding so mention routing and outbound identity metadata do not depend on inference from future events.',
+      });
+    }
+    if (wsState === 'connected' && latestEventIsStale) {
+      diagnostics.push({
+        code: 'lark_events_stale',
+        level: 'warn',
+        message: `Lark bot ${appId} has not stored an inbound event for ${Math.round((now - dbLastEventAt!) / 1000)}s.`,
+        hint: 'If a new Feishu message was just sent, Pal is not receiving provider events yet; repair the websocket, then check Feishu app event delivery.',
+      });
+    }
+    if (
+      wsState === 'connected'
+      && eventCount === 0
+      && dbEventCount > 0
+      && restartCount > 0
+      && lastRestartAt !== null
+      && now - lastRestartAt >= restartGraceMs
+    ) {
+      diagnostics.push({
+        code: 'lark_restart_ineffective',
+        level: 'warn',
+        message: `Lark bot ${appId} has not received provider events after ${restartCount} websocket restart(s).`,
+        hint: lastRestartReason === 'auto_stale'
+          ? 'Automatic websocket repair has already run without restoring Feishu events; inspect Feishu long-connection event subscription and other consumers of this app stream.'
+          : 'Manual websocket repair did not restore Feishu events; inspect Feishu long-connection event subscription and other consumers of this app stream.',
+      });
+    }
+    if (latestParseOk === false && !latestEventIsStale) {
+      diagnostics.push({
+        code: 'lark_raw_event_parse_errors',
+        level: 'warn',
+        message: `Lark bot ${appId} latest raw inbound event lacks a parseable provider event id (${dbParseErrorCount} total parse failure(s)).`,
+        hint: 'This is historical raw-event quality; prioritize live event freshness unless new provider events are arriving with parse_ok=0.',
+      });
+    }
+  }
+
+  const totals = input.deliveryWebSocket.totals;
+  if (totals.connections === 0) {
+    diagnostics.push({
+      code: 'delivery_no_connections',
+      level: 'warn',
+      message: 'No active daemon connections are registered.',
+      hint: 'Start or reconnect the daemon for this computer before expecting agent replies.',
+    });
+  } else if (totals.open_sockets === 0) {
+    diagnostics.push({
+      code: 'delivery_ws_no_open_sockets',
+      level: 'warn',
+      message: 'Daemon connections exist, but no delivery websocket is open.',
+      hint: 'Restart the daemon or check /api/daemon/ws connectivity and connection credentials.',
+    });
+  } else {
+    diagnostics.push({ code: 'delivery_ws_open', level: 'ok', message: `${totals.open_sockets} delivery websocket socket(s) are open.` });
+  }
+  const staleHeartbeatConnections = staleDeliveryHeartbeatConnections(input.deliveryWebSocket);
+  if (staleHeartbeatConnections.length > 0) {
+    diagnostics.push({
+      code: 'delivery_ws_heartbeat_stale',
+      level: 'warn',
+      message: `${staleHeartbeatConnections.length} open delivery websocket connection(s) have stale or missing pong heartbeats.`,
+      hint: 'The socket may be half-open. Restart the daemon or inspect websocket ping/pong logs and network path.',
+    });
+  }
+
+  if (totals.pending_deliveries > 0 || totals.claimed_deliveries > 0 || totals.processing_completed_deliveries > 0) {
+    diagnostics.push({
+      code: 'delivery_backlog_non_empty',
+      level: 'warn',
+      message: `Delivery backlog is non-empty: pending=${totals.pending_deliveries}, claimed=${totals.claimed_deliveries}, processing_completed=${totals.processing_completed_deliveries}.`,
+      hint: 'A message has reached the delivery queue; inspect daemon logs and runtime execution for the blocked deliveries.',
+    });
+  } else {
+    diagnostics.push({ code: 'delivery_backlog_empty', level: 'ok', message: 'Delivery backlog is empty.' });
+  }
+  if (totals.expired_active_deliveries > 0) {
+    diagnostics.push({
+      code: 'delivery_expired_active',
+      level: 'warn',
+      message: `${totals.expired_active_deliveries} claimed/processing delivery lease(s) are expired.`,
+      hint: 'Wait for retry or restart the affected daemon if the active run is gone.',
+    });
+  }
+  return diagnostics;
+}
+
+export function larkBotNeedsRestart(bot: LarkMessagingStatus, options: { now?: number; staleAfterMs?: number } = {}): boolean {
+  const wsState = stringField(bot.ws_state);
+  const lastError = stringField(bot.last_error);
+  if (lastError) return true;
+  if (wsState !== 'connected') return true;
+
+  const eventCount = numberField(bot.provider_event_count ?? bot.event_count);
+  const dbEventCount = providerDbEventCount(bot);
+  const hasDbLastEvent = Boolean(bot.db_last_event);
+  if (eventCount === 0 && dbEventCount > 0 && hasDbLastEvent) return true;
+
+  const dbLastEventAt = parseInboundReceivedAt(bot.db_last_event);
+  const staleAfterMs = options.staleAfterMs ?? larkEventStaleAfterMs();
+  const now = options.now ?? Date.now();
+  return dbLastEventAt !== null && now - dbLastEventAt > staleAfterMs;
+}
+
+export function selectRestartableLarkBotAppIds(bots: LarkMessagingStatus[], options: { now?: number; staleAfterMs?: number } = {}): string[] {
+  const selected = new Set<string>();
+  for (const bot of bots) {
+    const appId = stringField(bot.app_id);
+    if (!appId) continue;
+    if (larkBotNeedsRestart(bot, options)) selected.add(appId);
+  }
+  return Array.from(selected);
+}
+
+export function summarizeMessagingDiagnostics(diagnostics: MessagingDiagnostic[]): MessagingStatusSummary {
+  const warnCount = diagnostics.filter((diagnostic) => diagnostic.level === 'warn').length;
+  return {
+    level: warnCount > 0 ? 'warn' : 'ok',
+    codes: diagnostics.map((diagnostic) => diagnostic.code),
+    warn_count: warnCount,
+  };
+}
+
+export function buildMessagingHealth(status: MessagingStatus): MessagingHealth {
+  const larkWarningPriority = [
+    'lark_error',
+    'lark_ws_not_connected',
+    'lark_restart_ineffective',
+    'lark_probe_only_since_restart',
+    'lark_no_events_since_restart',
+    'lark_events_stale',
+    'lark_no_bots',
+    'lark_no_bound_agent',
+    'lark_no_authorized_users',
+    'lark_bot_open_id_unknown',
+    'lark_raw_event_parse_errors',
+  ];
+  const larkWarnings = prioritizedDiagnostics(warningDiagnostics(status.diagnostics, [
+    'lark_no_bots',
+    'lark_error',
+    'lark_ws_not_connected',
+    'lark_no_events_since_restart',
+    'lark_probe_only_since_restart',
+    'lark_events_stale',
+    'lark_restart_ineffective',
+    'lark_raw_event_parse_errors',
+    'lark_no_bound_agent',
+    'lark_no_authorized_users',
+    'lark_bot_open_id_unknown',
+  ]), larkWarningPriority);
+  const deliveryWsWarnings = warningDiagnostics(status.diagnostics, [
+    'delivery_no_connections',
+    'delivery_ws_no_open_sockets',
+    'delivery_ws_heartbeat_stale',
+  ]);
+  const backlogWarnings = warningDiagnostics(status.diagnostics, [
+    'delivery_backlog_non_empty',
+    'delivery_expired_active',
+  ]);
+  const totals = status.delivery_websocket.totals;
+  const staleHeartbeatConnections = staleDeliveryHeartbeatConnections(status.delivery_websocket);
+  const providerEvents = status.lark.bots.reduce((sum, bot) => sum + (bot.provider_event_count ?? bot.event_count ?? 0), 0);
+  const totalEvents = status.lark.bots.reduce((sum, bot) => sum + (bot.event_count ?? bot.provider_event_count ?? 0), 0);
+  const dbProviderEvents = status.lark.bots.reduce((sum, bot) => sum + providerDbEventCount(bot), 0);
+  const dbProbeEvents = status.lark.bots.reduce((sum, bot) => sum + (bot.db_probe_event_count ?? 0), 0);
+  const latestProviderEvent = status.lark.bots
+    .map((bot) => latestEventReceivedAt(bot))
+    .filter((value): value is string => Boolean(value))
+    .sort()
+    .at(-1) ?? null;
+  const latestProbeEvent = status.lark.bots
+    .map((bot) => latestProbeEventReceivedAt(bot))
+    .filter((value): value is string => Boolean(value))
+    .sort()
+    .at(-1) ?? null;
+  const larkCodes = larkWarnings.map((diagnostic) => diagnostic.code);
+  const deliveryWsCodes = deliveryWsWarnings.map((diagnostic) => diagnostic.code);
+  const backlogCodes = backlogWarnings.map((diagnostic) => diagnostic.code);
+
+  return {
+    level: status.summary.level,
+    domains: {
+      lark_provider: {
+        level: larkWarnings.length > 0 ? 'warn' : 'ok',
+        codes: larkCodes,
+        message: larkWarnings.length > 0
+          ? 'Lark provider ingress is not healthy.'
+          : 'Lark provider ingress has no active diagnostics.',
+        hint: larkProviderHint(larkCodes, larkWarnings),
+        recommended_action: recommendLarkProviderAction(larkCodes),
+        details: {
+          bots: status.lark.bots.length,
+          provider_events: providerEvents,
+          total_events: totalEvents,
+          db_provider_events: dbProviderEvents,
+          db_probe_events: dbProbeEvents,
+          latest_provider_event_at: latestProviderEvent,
+          latest_probe_event_at: latestProbeEvent,
+          restart_count: status.lark.bots.reduce((sum, bot) => sum + (bot.restart_count ?? 0), 0),
+          latest_restart_at: status.lark.bots
+            .map((bot) => bot.last_restart_at)
+            .filter((value): value is string => Boolean(value))
+            .sort()
+            .at(-1) ?? null,
+        },
+      },
+      delivery_websocket: {
+        level: deliveryWsWarnings.length > 0 ? 'warn' : 'ok',
+        codes: deliveryWsCodes,
+        message: deliveryWsWarnings.length > 0
+          ? 'Server-to-daemon delivery websocket is not healthy.'
+          : 'Server-to-daemon delivery websocket is healthy.',
+        hint: firstHint(deliveryWsWarnings),
+        recommended_action: recommendedAction(deliveryWsCodes, [
+          ['delivery_no_connections', 'start_or_reconnect_daemon'],
+          ['delivery_ws_no_open_sockets', 'restart_daemon_websocket'],
+          ['delivery_ws_heartbeat_stale', 'restart_daemon_websocket'],
+        ]),
+        details: {
+          connections: totals.connections,
+          open_sockets: totals.open_sockets,
+          stale_heartbeat_connections: staleHeartbeatConnections.length,
+          stale_heartbeat_connection_ids: staleHeartbeatConnections.map((connection) => connection.connection_id),
+          stale_heartbeat_computers: staleHeartbeatConnections.map((connection) => connection.computer_id ?? '-'),
+        },
+      },
+      delivery_backlog: {
+        level: backlogWarnings.length > 0 ? 'warn' : 'ok',
+        codes: backlogCodes,
+        message: backlogWarnings.length > 0
+          ? 'Delivery backlog has pending or active work.'
+          : 'Delivery backlog is empty.',
+        hint: firstHint(backlogWarnings),
+        recommended_action: recommendedAction(backlogCodes, [
+          ['delivery_backlog_non_empty', 'inspect_daemon_runtime'],
+          ['delivery_expired_active', 'restart_affected_daemon'],
+        ]),
+        details: {
+          pending: totals.pending_deliveries,
+          claimed: totals.claimed_deliveries,
+          processing_completed: totals.processing_completed_deliveries,
+          expired_active: totals.expired_active_deliveries,
+        },
+      },
+    },
+    summary: status.summary,
+  };
+}
+
+export function buildLarkProviderHealthLogLine(status: MessagingStatus): MessagingHealthLogLine | null {
+  return buildMessagingHealthLogLines(status).find((line) => line.throttle_key.startsWith('lark_provider:')) ?? null;
+}
+
+function formatHealthDetails(details: Record<string, unknown> | undefined): string[] {
+  if (!details) return [];
+  return Object.entries(details).map(([key, value]) => `${key}=${value ?? '-'}`);
+}
+
+function buildHealthLogLine(domainName: MessagingHealthDomainName, domain: MessagingHealthDomain): MessagingHealthLogLine | null {
+  if (domain.level !== 'warn') return null;
+  const parts = [
+    domain.message,
+    `domain=${domainName}`,
+    `codes=${domain.codes.length > 0 ? domain.codes.join(',') : '-'}`,
+    `action=${domain.recommended_action ?? '-'}`,
+    ...formatHealthDetails(domain.details),
+  ];
+  if (domain.hint) parts.push(`hint=${domain.hint}`);
+  return {
+    throttle_key: `${domainName}:${domain.codes.join(',')}:${domain.recommended_action ?? ''}`,
+    message: parts.join(' '),
+  };
+}
+
+export function buildMessagingHealthLogLines(status: MessagingStatus): MessagingHealthLogLine[] {
+  const health = buildMessagingHealth(status);
+  return (Object.entries(health.domains) as Array<[MessagingHealthDomainName, MessagingHealthDomain]>)
+    .map(([name, domain]) => buildHealthLogLine(name, domain))
+    .filter((line): line is MessagingHealthLogLine => Boolean(line));
+}

package/src/migrations/024_agents_model.ts

@@ -0,0 +1,10 @@
+import type { Database } from 'bun:sqlite';
+
+export const version = 24;
+export const name = 'agents_model';
+
+export function up(db: Database): void {
+  db.exec(`
+    ALTER TABLE agents ADD COLUMN model TEXT NULL;
+  `);
+}