@controlflow-ai/daemon 0.1.2 → 0.1.3

package/src/messaging-cli.ts

@@ -0,0 +1,741 @@
+import { boolFlag, flag, numberFlag } from './args.js';
+import { LockClient } from './client.js';
+import { defaultLarkConfigPath, loadLarkCredentials } from './lark/credentials.js';
+import { resolveLarkBotInfo, type LarkBotInfoResult } from './lark/setup.js';
+import { recommendLarkProviderAction, selectRestartableLarkBotAppIds } from './messaging-status.js';
+
+export interface MessagingCliArgs {
+  values: string[];
+  flags: Record<string, string | boolean>;
+}
+
+interface LarkDoctorCheck {
+  code: string;
+  level: 'ok' | 'warn';
+  message: string;
+  hint?: string;
+}
+
+interface LarkDoctorBot {
+  app_id: string;
+  label: string | null;
+  agent: string | null;
+  recommended_action?: string;
+  checks: LarkDoctorCheck[];
+}
+
+function printJson(value: unknown): void {
+  console.log(JSON.stringify(value, null, 2));
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function larkConfigPath(flags: Record<string, string | boolean>): string {
+  return flag(flags, 'lark-config') ?? flag(flags, 'config') ?? defaultLarkConfigPath();
+}
+
+async function resolveBotInfoForDoctor(appId: string, appSecret: string, options: { budgetMs?: number } = {}): Promise<LarkBotInfoResult> {
+  if (process.env.NODE_ENV === 'test' && process.env.PAL_TEST_LARK_BOT_OPEN_ID) {
+    return { ok: true, openId: process.env.PAL_TEST_LARK_BOT_OPEN_ID };
+  }
+  return resolveLarkBotInfo(appId, appSecret, options);
+}
+
+function dbProviderEventCount(bot: { db_event_count?: number; db_provider_event_count?: number }): number {
+  return bot.db_provider_event_count ?? bot.db_event_count ?? 0;
+}
+
+function larkEventMarker(bot: { event_count?: number; provider_event_count?: number; db_event_count?: number; db_provider_event_count?: number; db_last_event?: unknown }) {
+  const dbLastEvent = bot.db_last_event && typeof bot.db_last_event === 'object'
+    ? bot.db_last_event as { id?: unknown; event_id?: unknown; received_at?: unknown; parse_ok?: unknown }
+    : null;
+  return {
+    event_count: bot.provider_event_count ?? bot.event_count ?? 0,
+    db_event_count: dbProviderEventCount(bot),
+    db_last_event_id: typeof dbLastEvent?.id === 'string' ? dbLastEvent.id : null,
+    db_last_provider_event_id: typeof dbLastEvent?.event_id === 'string' ? dbLastEvent.event_id : null,
+    db_last_received_at: typeof dbLastEvent?.received_at === 'string' ? dbLastEvent.received_at : null,
+    db_last_parse_ok: dbLastEvent?.parse_ok === 1 || dbLastEvent?.parse_ok === true ? true : dbLastEvent?.parse_ok === 0 || dbLastEvent?.parse_ok === false ? false : null,
+  };
+}
+
+function sameLarkEventMarker(a: ReturnType<typeof larkEventMarker>, b: ReturnType<typeof larkEventMarker>): boolean {
+  return a.event_count === b.event_count
+    && a.db_event_count === b.db_event_count
+    && a.db_last_event_id === b.db_last_event_id
+    && a.db_last_provider_event_id === b.db_last_provider_event_id
+    && a.db_last_received_at === b.db_last_received_at
+    && a.db_last_parse_ok === b.db_last_parse_ok;
+}
+
+function larkWatchSnapshot(status: Awaited<ReturnType<LockClient['getMessagingStatus']>>, appId: string) {
+  const bot = status.lark.bots.find((candidate) => candidate.app_id === appId);
+  if (!bot) throw new Error(`Lark bot not found: ${appId}`);
+  const marker = larkEventMarker(bot);
+  const diagnostics = status.diagnostics
+    .filter((diagnostic) => diagnostic.message.includes(appId))
+    .map((diagnostic) => diagnostic.code);
+  return {
+    marker,
+    ws_state: bot.ws_state ?? null,
+    provider_event_count: bot.provider_event_count ?? bot.event_count ?? 0,
+    total_event_count: bot.event_count ?? bot.provider_event_count ?? 0,
+    provider_message_event_count: bot.provider_message_event_count ?? bot.message_event_count ?? 0,
+    total_message_event_count: bot.message_event_count ?? bot.provider_message_event_count ?? 0,
+    last_provider_event_at: bot.last_provider_event_at ?? null,
+    last_provider_message_event_at: bot.last_provider_message_event_at ?? null,
+    diagnostics,
+  };
+}
+
+function formatMaybe(value: string | number | null | undefined): string {
+  return value === null || value === undefined || value === '' ? '-' : String(value);
+}
+
+function formatHealthDetailValue(value: unknown): string {
+  if (Array.isArray(value)) return value.length ? value.map((item) => formatHealthDetailValue(item)).join(',') : '[]';
+  if (value && typeof value === 'object') return JSON.stringify(value);
+  return formatMaybe(value as string | number | null | undefined);
+}
+
+function isTerminalDeliveryStatus(status: string): boolean {
+  return status === 'acked' || status === 'failed' || status === 'canceled';
+}
+
+function formatLarkDbLastEvent(bot: { db_last_event?: unknown }): string | null {
+  if (!bot.db_last_event || typeof bot.db_last_event !== 'object') return null;
+  const event = bot.db_last_event as {
+    received_at?: unknown;
+    event_id?: unknown;
+    event_type?: unknown;
+    parse_ok?: unknown;
+  };
+  const parseOk = event.parse_ok === 1 || event.parse_ok === true
+    ? '1'
+    : event.parse_ok === 0 || event.parse_ok === false
+      ? '0'
+      : '-';
+  return `latest_event=${formatMaybe(typeof event.event_type === 'string' ? event.event_type : null)} id=${formatMaybe(typeof event.event_id === 'string' ? event.event_id : null)} at=${formatMaybe(typeof event.received_at === 'string' ? event.received_at : null)} parse_ok=${parseOk}`;
+}
+
+async function larkIngressEvidence(serverClient: LockClient, appId: string, recentLimit: number) {
+  const [status, health, recent] = await Promise.all([
+    serverClient.getMessagingStatus(),
+    serverClient.getMessagingHealth(),
+    serverClient.listRecentLarkEvents(recentLimit),
+  ]);
+  const bot = status.lark.bots.find((candidate) => candidate.app_id === appId);
+  return {
+    status,
+    health,
+    recent,
+    bot: bot ?? null,
+    snapshot: bot ? larkWatchSnapshot(status, appId) : null,
+    marker: bot ? larkEventMarker(bot) : null,
+  };
+}
+
+function printLarkIngressEvidence(input: Awaited<ReturnType<typeof larkIngressEvidence>>): void {
+  const lark = input.health.domains.lark_provider;
+  console.log(`  lark_provider=${lark.level} codes=${lark.codes.join(',') || '-'} action=${lark.recommended_action ?? '-'}`);
+  if (lark.details) {
+    const details = Object.entries(lark.details).map(([key, value]) => `${key}=${formatHealthDetailValue(value)}`).join(' ');
+    if (details) console.log(`    ${details}`);
+  }
+  if (lark.hint) console.log(`    hint: ${lark.hint}`);
+  const recent = input.recent.slice(0, 5);
+  if (recent.length === 0) {
+    console.log('  recent_lark_events=-');
+    return;
+  }
+  console.log('  recent_lark_events:');
+  for (const event of recent) {
+    console.log(`    ${event.received_at} app=${event.app_id} type=${event.event_type} id=${event.event_id} probe=${event.is_probe ? 'true' : 'false'} parse_ok=${event.parse_ok}`);
+  }
+}
+
+async function printMessagingStatus(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const status = await serverClient.getMessagingStatus();
+  if (boolFlag(flags, 'json')) {
+    printJson(status);
+    return;
+  }
+  const verbose = boolFlag(flags, 'verbose');
+  console.log(`messaging: ${status.summary.level} warnings=${status.summary.warn_count} codes=${status.summary.codes.join(',') || '-'}`);
+  for (const diagnostic of status.diagnostics) {
+    console.log(`  ${diagnostic.level} ${diagnostic.code}: ${diagnostic.message}`);
+    if (diagnostic.hint) console.log(`    hint: ${diagnostic.hint}`);
+  }
+  for (const bot of status.lark.bots) {
+    const providerEvents = bot.provider_event_count ?? bot.event_count ?? 0;
+    const totalEvents = bot.event_count ?? providerEvents;
+    const eventSummary = typeof bot.provider_event_count === 'number'
+      ? `provider_events=${providerEvents} events=${totalEvents}`
+      : `events=${providerEvents}`;
+    console.log(`  lark ${bot.app_id} ws=${bot.ws_state ?? '-'} agent=${bot.agent ?? '-'} authorized_users=${bot.authorized_user_count ?? 0} bot_open_id=${bot.bot_open_id_known ? 'known' : 'unknown'} ${eventSummary} db_provider_events=${dbProviderEventCount(bot)} db_probe_events=${bot.db_probe_event_count ?? 0} parse_errors=${bot.db_parse_error_count ?? 0}`);
+    if (bot.config_path) console.log(`    config=${bot.config_path}`);
+    if (bot.db_path) console.log(`    db=${bot.db_path}`);
+    const latestEvent = verbose ? formatLarkDbLastEvent(bot) : null;
+    if (latestEvent) console.log(`    ${latestEvent}`);
+    if (verbose && (bot.last_provider_event_at || bot.last_provider_message_event_at)) {
+      console.log(`    provider_event_at=${formatMaybe(bot.last_provider_event_at)} provider_message_at=${formatMaybe(bot.last_provider_message_event_at)}`);
+    }
+    if (verbose && (bot.ws_last_event_type || bot.ws_last_error)) {
+      console.log(`    ws_event=${formatMaybe(bot.ws_last_event_type)} at=${formatMaybe(bot.ws_last_event_at)} error=${formatMaybe(bot.ws_last_error)}`);
+    }
+    if (verbose && ((bot.restart_count ?? 0) > 0 || bot.last_restart_at || bot.last_restart_reason)) {
+      console.log(`    restarts=${bot.restart_count ?? 0} last_restart=${formatMaybe(bot.last_restart_at)} reason=${formatMaybe(bot.last_restart_reason)}`);
+    }
+  }
+  const totals = status.delivery_websocket.totals;
+  console.log(`  delivery_ws connections=${totals.connections} open_sockets=${totals.open_sockets} pending=${totals.pending_deliveries} claimed=${totals.claimed_deliveries} processing_completed=${totals.processing_completed_deliveries} expired_active=${totals.expired_active_deliveries}`);
+  if (verbose) {
+    for (const connection of status.delivery_websocket.connections) {
+      console.log(`    connection ${connection.connection_id} computer=${formatMaybe(connection.computer_id)} open_sockets=${connection.open_sockets} last_open=${formatMaybe(connection.last_open_at)} last_ping=${formatMaybe(connection.last_ping_at)} last_pong=${formatMaybe(connection.last_pong_at)} last_close=${formatMaybe(connection.last_close_at)} close_code=${formatMaybe(connection.last_close_code)}`);
+      const pendingAgents = connection.pending_agents.map((item) => `${item.agent}:${item.pending}`).join(',') || '-';
+      const backlog = connection.backlog
+        .map((item) => `${item.agent}:pending=${item.pending},claimed=${item.claimed},processing_completed=${item.processing_completed},expired=${item.expired_active}`)
+        .join(';') || '-';
+      console.log(`      pending_agents=${pendingAgents}`);
+      console.log(`      backlog=${backlog}`);
+      if (connection.last_close_reason) console.log(`      close_reason=${connection.last_close_reason}`);
+    }
+  }
+}
+
+async function printMessagingHealth(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const health = await serverClient.getMessagingHealth();
+  if (boolFlag(flags, 'json')) {
+    printJson(health);
+    return;
+  }
+  console.log(`messaging health: ${health.level} warnings=${health.summary.warn_count} codes=${health.summary.codes.join(',') || '-'}`);
+  for (const [name, domain] of Object.entries(health.domains)) {
+    const codes = domain.codes.join(',') || '-';
+    const details = domain.details
+      ? Object.entries(domain.details).map(([key, value]) => `${key}=${formatHealthDetailValue(value)}`).join(' ')
+      : '';
+    console.log(`  ${domain.level} ${name}: ${domain.message} codes=${codes}${domain.recommended_action ? ` action=${domain.recommended_action}` : ''}${details ? ` ${details}` : ''}`);
+    if (domain.hint) console.log(`    hint: ${domain.hint}`);
+  }
+}
+
+async function watchLarkMessaging(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const appId = flag(flags, 'app-id') ?? flag(flags, 'id');
+  const timeoutMs = numberFlag(flags, 'timeout-ms', 60_000)!;
+  const intervalMs = numberFlag(flags, 'interval-ms', 1_000)!;
+  const json = boolFlag(flags, 'json');
+  if (timeoutMs <= 0) throw new Error('--timeout-ms must be positive');
+  if (intervalMs <= 0) throw new Error('--interval-ms must be positive');
+
+  const initial = await serverClient.getMessagingStatus();
+  const initialBot = appId
+    ? initial.lark.bots.find((bot) => bot.app_id === appId)
+    : initial.lark.bots[0];
+  if (!initialBot) throw new Error(appId ? `Lark bot not found: ${appId}` : 'No Lark bot is active on this server');
+  const targetAppId = initialBot.app_id;
+  const baseline = larkEventMarker(initialBot);
+  const baselineSnapshot = larkWatchSnapshot(initial, targetAppId);
+  const startedAt = Date.now();
+  let polls = 0;
+  let latest = initial;
+  let latestMarker = baseline;
+  let latestSnapshot = baselineSnapshot;
+
+  while (Date.now() - startedAt < timeoutMs) {
+    await sleep(Math.min(intervalMs, Math.max(1, timeoutMs - (Date.now() - startedAt))));
+    polls += 1;
+    latest = await serverClient.getMessagingStatus();
+    const bot = latest.lark.bots.find((candidate) => candidate.app_id === targetAppId);
+    if (!bot) throw new Error(`Lark bot disappeared while watching: ${targetAppId}`);
+    latestMarker = larkEventMarker(bot);
+    latestSnapshot = larkWatchSnapshot(latest, targetAppId);
+    if (!sameLarkEventMarker(baseline, latestMarker)) {
+      if (json) {
+        printJson({ observed: true, app_id: targetAppId, polls, baseline, latest: latestMarker, baseline_status: baselineSnapshot, latest_status: latestSnapshot, summary: latest.summary });
+      } else {
+        console.log(`Lark provider event observed. app=${targetAppId} polls=${polls} provider_events=${latestSnapshot.provider_event_count} total_events=${latestSnapshot.total_event_count} db_events=${latestMarker.db_event_count} latest=${latestMarker.db_last_provider_event_id ?? latestMarker.db_last_event_id ?? '-'} latest_at=${latestMarker.db_last_received_at ?? '-'}`);
+      }
+      return;
+    }
+  }
+
+  if (json) {
+    printJson({ observed: false, app_id: targetAppId, polls, timeout_ms: timeoutMs, baseline, latest: latestMarker, baseline_status: baselineSnapshot, latest_status: latestSnapshot, summary: latest.summary });
+    return;
+  }
+  const codes = latestSnapshot.diagnostics.join(',') || '-';
+  throw new Error(`Timed out waiting for Lark provider event for ${targetAppId} after ${timeoutMs}ms; provider_events=${latestSnapshot.provider_event_count} total_events=${latestSnapshot.total_event_count} latest=${latestMarker.db_last_provider_event_id ?? latestMarker.db_last_event_id ?? '-'} latest_at=${latestMarker.db_last_received_at ?? '-'} diagnostics=${codes}`);
+}
+
+async function verifyLarkIngressMessaging(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const appId = flag(flags, 'app-id') ?? flag(flags, 'id');
+  const timeoutMs = numberFlag(flags, 'timeout-ms', 60_000)!;
+  const intervalMs = numberFlag(flags, 'interval-ms', 1_000)!;
+  const recentLimit = numberFlag(flags, 'recent-limit', 5)!;
+  const json = boolFlag(flags, 'json');
+  if (timeoutMs <= 0) throw new Error('--timeout-ms must be positive');
+  if (intervalMs <= 0) throw new Error('--interval-ms must be positive');
+  if (recentLimit <= 0) throw new Error('--recent-limit must be positive');
+
+  const initial = await serverClient.getMessagingStatus();
+  const initialBot = appId
+    ? initial.lark.bots.find((bot) => bot.app_id === appId)
+    : initial.lark.bots[0];
+  if (!initialBot) throw new Error(appId ? `Lark bot not found: ${appId}` : 'No Lark bot is active on this server');
+  const targetAppId = initialBot.app_id;
+  const baseline = larkEventMarker(initialBot);
+  const baselineSnapshot = larkWatchSnapshot(initial, targetAppId);
+  const startedAt = Date.now();
+  let polls = 0;
+  let latest = initial;
+  let latestMarker = baseline;
+  let latestSnapshot = baselineSnapshot;
+
+  if (!json) {
+    console.log(`Lark ingress verification started. app=${targetAppId} timeout_ms=${timeoutMs}`);
+    console.log('  Send one real Feishu message to this bot now; local probe events are ignored.');
+    console.log(`  baseline provider_events=${baselineSnapshot.provider_event_count} db_events=${baseline.db_event_count} latest=${baseline.db_last_provider_event_id ?? baseline.db_last_event_id ?? '-'} latest_at=${baseline.db_last_received_at ?? '-'}`);
+  }
+
+  while (Date.now() - startedAt < timeoutMs) {
+    await sleep(Math.min(intervalMs, Math.max(1, timeoutMs - (Date.now() - startedAt))));
+    polls += 1;
+    latest = await serverClient.getMessagingStatus();
+    const bot = latest.lark.bots.find((candidate) => candidate.app_id === targetAppId);
+    if (!bot) throw new Error(`Lark bot disappeared while verifying ingress: ${targetAppId}`);
+    latestMarker = larkEventMarker(bot);
+    latestSnapshot = larkWatchSnapshot(latest, targetAppId);
+    if (!sameLarkEventMarker(baseline, latestMarker)) {
+      const evidence = await larkIngressEvidence(serverClient, targetAppId, recentLimit);
+      const payload = {
+        observed: true,
+        app_id: targetAppId,
+        polls,
+        baseline,
+        latest: latestMarker,
+        baseline_status: baselineSnapshot,
+        latest_status: latestSnapshot,
+        health: evidence.health,
+        recent_events: evidence.recent,
+        summary: latest.summary,
+      };
+      if (json) {
+        printJson(payload);
+        return;
+      }
+      console.log(`Lark provider ingress verified. app=${targetAppId} polls=${polls} provider_events=${latestSnapshot.provider_event_count} total_events=${latestSnapshot.total_event_count} latest=${latestMarker.db_last_provider_event_id ?? latestMarker.db_last_event_id ?? '-'} latest_at=${latestMarker.db_last_received_at ?? '-'}`);
+      printLarkIngressEvidence(evidence);
+      return;
+    }
+  }
+
+  const evidence = await larkIngressEvidence(serverClient, targetAppId, recentLimit);
+  const payload = {
+    observed: false,
+    app_id: targetAppId,
+    polls,
+    timeout_ms: timeoutMs,
+    baseline,
+    latest: latestMarker,
+    baseline_status: baselineSnapshot,
+    latest_status: latestSnapshot,
+    health: evidence.health,
+    recent_events: evidence.recent,
+    summary: latest.summary,
+  };
+  if (json) {
+    printJson(payload);
+    return;
+  }
+  console.log(`No real Feishu provider event observed. app=${targetAppId} timeout_ms=${timeoutMs} polls=${polls}`);
+  console.log(`  latest provider_events=${latestSnapshot.provider_event_count} db_events=${latestMarker.db_event_count} latest=${latestMarker.db_last_provider_event_id ?? latestMarker.db_last_event_id ?? '-'} latest_at=${latestMarker.db_last_received_at ?? '-'}`);
+  printLarkIngressEvidence(evidence);
+  throw new Error('Lark provider ingress verification failed; inspect Feishu long-connection event subscription or another consumer of this app stream.');
+}
+
+async function printRecentLarkEvents(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const limit = numberFlag(flags, 'limit', 20)!;
+  if (limit <= 0) throw new Error('--limit must be positive');
+  const events = await serverClient.listRecentLarkEvents(limit);
+  if (boolFlag(flags, 'json')) {
+    printJson({ events });
+    return;
+  }
+  if (events.length === 0) {
+    console.log('No recent Lark inbound events.');
+    return;
+  }
+  for (const event of events) {
+    console.log(`${event.received_at} app=${event.app_id} type=${event.event_type} id=${event.event_id} parse_ok=${event.parse_ok} bytes=${event.bytes}${event.is_probe ? ' probe=true' : ''}`);
+  }
+}
+
+async function repairLarkEventParseFailures(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const appId = flag(flags, 'app-id') ?? flag(flags, 'id');
+  const limit = numberFlag(flags, 'limit', 100)!;
+  if (limit <= 0) throw new Error('--limit must be positive');
+  const dryRun = !boolFlag(flags, 'apply');
+  const result = await serverClient.repairLarkEventParseFailures({ appId, limit, dryRun });
+  if (boolFlag(flags, 'json')) {
+    printJson(result);
+    return;
+  }
+  const mode = result.dry_run ? 'dry-run' : 'applied';
+  console.log(`Lark raw event parse repair ${mode}: scanned=${result.scanned} repaired=${result.repaired} unchanged=${result.unchanged} conflicts=${result.conflicts} errors=${result.errors}`);
+  for (const row of result.rows.slice(0, 20)) {
+    console.log(`  ${row.status} app=${row.app_id} id=${row.old_event_id} -> ${row.new_event_id} type=${row.old_event_type} -> ${row.new_event_type}`);
+    if (row.error) console.log(`    error=${row.error}`);
+  }
+  if (result.rows.length > 20) {
+    console.log(`  ... ${result.rows.length - 20} more row(s) omitted`);
+  }
+}
+
+async function probeDeliveryMessaging(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const agent = flag(flags, 'agent') ?? flag(flags, 'to') ?? 'lock';
+  const room = flag(flags, 'room') ?? flag(flags, 'chat');
+  const sender = flag(flags, 'sender') ?? flag(flags, 'from');
+  const content = flag(flags, 'text') ?? flag(flags, 'content');
+  const timeoutMs = numberFlag(flags, 'timeout-ms', 120_000)!;
+  const intervalMs = numberFlag(flags, 'interval-ms', 1_000)!;
+  if (timeoutMs < 0) throw new Error('--timeout-ms must be zero or positive');
+  if (intervalMs <= 0) throw new Error('--interval-ms must be positive');
+
+  const created = await serverClient.probeDelivery({ agent, room, sender, content });
+  const deliveryIds = created.deliveries.map((delivery) => delivery.id);
+  const payload = {
+    created,
+    final_deliveries: created.deliveries,
+    replies: [] as Awaited<ReturnType<LockClient['getMessages']>>,
+    timed_out: false,
+  };
+
+  if (deliveryIds.length > 0 && timeoutMs > 0) {
+    const startedAt = Date.now();
+    while (Date.now() - startedAt < timeoutMs) {
+      const deliveries = await Promise.all(deliveryIds.map((id) => serverClient.getDelivery(id)));
+      payload.final_deliveries = deliveries;
+      if (deliveries.every((delivery) => isTerminalDeliveryStatus(delivery.status))) break;
+      await sleep(Math.min(intervalMs, Math.max(1, timeoutMs - (Date.now() - startedAt))));
+    }
+    if (!payload.final_deliveries.every((delivery) => isTerminalDeliveryStatus(delivery.status))) {
+      payload.timed_out = true;
+    }
+  } else if (deliveryIds.length > 0) {
+    payload.final_deliveries = await Promise.all(deliveryIds.map((id) => serverClient.getDelivery(id)));
+  }
+
+  const params = new URLSearchParams({ chat: created.probe.room, q: created.probe.token, limit: '20' });
+  payload.replies = (await serverClient.getMessages(params)).filter((message) => message.id !== created.message.id);
+
+  if (boolFlag(flags, 'json')) {
+    printJson(payload);
+    return;
+  }
+
+  console.log(`Delivery probe created. agent=${created.probe.agent} room=${created.probe.room} message=${created.message.id} deliveries=${deliveryIds.join(',') || '-'}`);
+  console.log(`  notify deliveries=${created.notify.deliveries} target_connections=${created.notify.target_connections} open_sockets=${created.notify.open_sockets} websocket_frames=${created.notify.websocket_frames}`);
+  for (const delivery of payload.final_deliveries) {
+    console.log(`  delivery ${delivery.id} status=${delivery.status} attempts=${delivery.attempts} run=${delivery.run_id ?? '-'} error=${delivery.last_error || '-'}`);
+  }
+  if (payload.replies.length > 0) {
+    for (const reply of payload.replies) {
+      console.log(`  reply message=${reply.id} sender=${reply.sender} recipient=${reply.recipient ?? '-'} content=${reply.content}`);
+    }
+  }
+  if (payload.timed_out) {
+    throw new Error(`delivery probe timed out after ${timeoutMs}ms`);
+  }
+  if (payload.final_deliveries.length === 0) {
+    throw new Error('delivery probe created no deliveries');
+  }
+  const failed = payload.final_deliveries.find((delivery) => delivery.status !== 'acked');
+  if (failed) {
+    throw new Error(`delivery probe did not ack delivery ${failed.id}; status=${failed.status}`);
+  }
+}
+
+async function repairLarkMessaging(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const dryRun = boolFlag(flags, 'dry-run');
+  const json = boolFlag(flags, 'json');
+  const status = await serverClient.getMessagingStatus();
+  const appIds = selectRestartableLarkBotAppIds(status.lark.bots);
+  const result = dryRun || appIds.length === 0
+    ? null
+    : await serverClient.restartLarkWebSocket({ appIds });
+
+  if (json) {
+    printJson({
+      dry_run: dryRun,
+      restart_app_ids: appIds,
+      restarted: result?.restarted ?? [],
+      missing: result?.missing ?? [],
+      summary: status.summary,
+    });
+    return;
+  }
+
+  if (appIds.length === 0) {
+    console.log('No Lark websocket repair needed.');
+    return;
+  }
+  if (dryRun) {
+    console.log(`Lark websocket repair would restart: ${appIds.join(',')}`);
+    return;
+  }
+  console.log(`Lark websocket repair requested. restarted=${result?.restarted.join(',') || '-'} missing=${result?.missing.join(',') || '-'}`);
+}
+
+function pushCheck(checks: LarkDoctorCheck[], check: LarkDoctorCheck): void {
+  checks.push(check);
+}
+
+function statusDiagnosticsForApp(status: Awaited<ReturnType<LockClient['getMessagingStatus']>>, appId: string) {
+  return status.diagnostics.filter((diagnostic) => diagnostic.message.includes(appId));
+}
+
+async function doctorLarkMessaging(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const appId = flag(flags, 'app-id') ?? flag(flags, 'id');
+  const configPath = larkConfigPath(flags);
+  const json = boolFlag(flags, 'json');
+  const budgetMs = numberFlag(flags, 'budget-ms', 10_000)!;
+  const store = loadLarkCredentials(configPath);
+  const targets = appId ? store.bots.filter((bot) => bot.appId === appId) : store.bots;
+  if (targets.length === 0) throw new Error(appId ? `No Lark bot found in ${configPath}: ${appId}` : `No Lark bots configured in ${configPath}`);
+
+  const status = await serverClient.getMessagingStatus();
+  const bots: LarkDoctorBot[] = [];
+  let warnCount = 0;
+
+  for (const bot of targets) {
+    const checks: LarkDoctorCheck[] = [];
+    const serverBot = status.lark.bots.find((candidate) => candidate.app_id === bot.appId);
+    const agents = bot.agent?.trim() ? [bot.agent.trim()] : [];
+
+    pushCheck(checks, {
+      code: 'local_config_loaded',
+      level: 'ok',
+      message: `Local config contains app ${bot.appId} at ${configPath}.`,
+    });
+    pushCheck(checks, bot.appSecret?.trim()
+      ? { code: 'local_secret_present', level: 'ok', message: 'Local app secret is present (redacted).' }
+      : { code: 'local_secret_missing', level: 'warn', message: 'Local app secret is missing.', hint: 'Rebind this Lark bot with --lark-app-id and --lark-app-secret.' });
+    pushCheck(checks, agents.length > 0
+      ? { code: 'local_agent_bound', level: 'ok', message: `Bot is bound to agent ${agents.join(',')}.` }
+      : { code: 'local_agent_missing', level: 'warn', message: 'Bot is not bound to a Pal agent.', hint: 'Bind the bot to an agent before expecting Feishu messages to create work.' });
+
+    if (!serverBot) {
+      pushCheck(checks, {
+        code: 'server_bot_missing',
+        level: 'warn',
+        message: 'Server is not running this Lark bot.',
+        hint: 'Run messaging restart-lark or POST /api/lark/reload after saving the bot config.',
+      });
+    } else {
+      pushCheck(checks, serverBot.ws_state === 'connected'
+        ? { code: 'server_ws_connected', level: 'ok', message: 'Server Lark websocket is connected.' }
+        : { code: 'server_ws_not_connected', level: 'warn', message: `Server Lark websocket state is ${serverBot.ws_state ?? 'unknown'}.`, hint: 'Run messaging repair-lark and inspect server logs if reconnect fails.' });
+      pushCheck(checks, serverBot.authorized_user_count && serverBot.authorized_user_count > 0
+        ? { code: 'authorized_users_present', level: 'ok', message: `${serverBot.authorized_user_count} authorized Feishu user(s) are configured.` }
+        : { code: 'authorized_users_missing', level: 'warn', message: 'No authorized Feishu sender union IDs are configured.', hint: 'Add the sender with lark-users add, otherwise real events are stored but skipped.' });
+      const runtimeDiagnostics = statusDiagnosticsForApp(status, bot.appId);
+      for (const diagnostic of runtimeDiagnostics) {
+        if (diagnostic.level !== 'warn') continue;
+        pushCheck(checks, {
+          code: `runtime_${diagnostic.code}`,
+          level: 'warn',
+          message: diagnostic.message,
+          hint: diagnostic.hint,
+        });
+      }
+    }
+
+    if (bot.appSecret?.trim()) {
+      const botInfo = await resolveBotInfoForDoctor(bot.appId, bot.appSecret, { budgetMs });
+      if (botInfo.ok) {
+        pushCheck(checks, {
+          code: 'openapi_credentials_valid',
+          level: 'ok',
+          message: `OpenAPI credentials are valid${botInfo.appName ? ` for ${botInfo.appName}` : ''}.`,
+        });
+        if (bot.botOpenId?.trim()) {
+          pushCheck(checks, bot.botOpenId === botInfo.openId
+            ? { code: 'bot_open_id_matches', level: 'ok', message: 'Configured bot open_id matches OpenAPI bot info.' }
+            : { code: 'bot_open_id_mismatch', level: 'warn', message: 'Configured bot open_id does not match OpenAPI bot info.', hint: 'Rebind the Lark bot so mention routing uses the current app identity.' });
+        } else {
+          pushCheck(checks, {
+            code: 'bot_open_id_missing',
+            level: 'warn',
+            message: 'Local config does not store the bot open_id.',
+            hint: 'Rebind the Lark bot to cache bot open_id for reliable mention routing.',
+          });
+        }
+      } else {
+        pushCheck(checks, {
+          code: `openapi_${botInfo.error}`,
+          level: 'warn',
+          message: `OpenAPI credential check failed: ${botInfo.message}`,
+          hint: botInfo.error === 'invalid_credentials' ? 'Rebind the app with the current App ID and App Secret.' : 'Check network access to open.feishu.cn and retry.',
+        });
+      }
+    }
+
+    const stale = checks.some((check) => (
+      check.code === 'runtime_lark_events_stale'
+      || check.code === 'runtime_lark_no_events_since_restart'
+      || check.code === 'runtime_lark_probe_only_since_restart'
+      || check.code === 'runtime_lark_restart_ineffective'
+    ));
+    pushCheck(checks, {
+      code: 'platform_event_subscription_manual',
+      level: stale ? 'warn' : 'ok',
+      message: 'Feishu event subscription state cannot be verified through Pal local state.',
+      hint: stale
+        ? 'In Feishu developer console, verify this app uses long-connection event delivery and subscribes to im.message.receive_v1; also ensure no other environment is consuming the same app stream.'
+        : 'If real Feishu messages do not arrive, verify long-connection event delivery and im.message.receive_v1 in Feishu developer console.',
+    });
+    pushCheck(checks, {
+      code: 'platform_long_connection_cluster_mode',
+      level: stale ? 'warn' : 'ok',
+      message: 'Feishu long-connection delivery is cluster-mode: multiple clients for the same app share events instead of each receiving every event.',
+      hint: stale
+        ? 'Stop other local, demo, staging, or production websocket clients using this App ID while verifying ingress; otherwise a fresh message may be delivered to a different client.'
+        : 'Keep only the intended Pal server connected for this App ID when running deterministic E2E checks.',
+    });
+
+    const runtimeCodes = checks
+      .filter((check) => check.code.startsWith('runtime_'))
+      .map((check) => check.code.replace(/^runtime_/, ''));
+    const recommended = recommendLarkProviderAction(runtimeCodes);
+    warnCount += checks.filter((check) => check.level === 'warn').length;
+    bots.push({
+      app_id: bot.appId,
+      label: bot.label?.trim() || null,
+      agent: bot.agent?.trim() || null,
+      ...(recommended ? { recommended_action: recommended } : {}),
+      checks,
+    });
+  }
+
+  const payload = {
+    level: warnCount > 0 ? 'warn' : 'ok',
+    warn_count: warnCount,
+    config_path: configPath,
+    server: serverClient.baseUrl,
+    bots,
+  };
+  if (json) {
+    printJson(payload);
+    return;
+  }
+
+  console.log(`lark doctor: ${payload.level} warnings=${warnCount} config=${configPath} server=${serverClient.baseUrl}`);
+  for (const bot of bots) {
+    console.log(`  app=${bot.app_id} agent=${bot.agent ?? '-'} label=${bot.label ?? '-'}`);
+    if (bot.recommended_action) console.log(`    action: ${bot.recommended_action}`);
+    for (const check of bot.checks) {
+      console.log(`    ${check.level} ${check.code}: ${check.message}`);
+      if (check.hint) console.log(`      hint: ${check.hint}`);
+    }
+  }
+}
+
+async function probeLarkMessaging(serverClient: LockClient, flags: Record<string, string | boolean>): Promise<void> {
+  const appId = flag(flags, 'app-id') ?? flag(flags, 'id');
+  const senderUserId = flag(flags, 'sender-user-id') ?? flag(flags, 'user-id');
+  const chatId = flag(flags, 'chat-id');
+  if (!appId) throw new Error('--app-id is required');
+  if (!senderUserId) throw new Error('--sender-user-id is required');
+  if (!chatId) throw new Error('--chat-id is required');
+
+  const mentionOpenId = flag(flags, 'mention-open-id');
+  const text = flag(flags, 'text') ?? (mentionOpenId ? '@Probe hello from pal probe' : 'hello from pal probe');
+  const messageId = flag(flags, 'message-id') ?? `probe_${Date.now()}_${crypto.randomUUID()}`;
+  const data = {
+    sender: { sender_id: { open_id: flag(flags, 'sender-open-id') ?? `probe_${senderUserId}`, union_id: senderUserId }, sender_type: 'user' },
+    message: {
+      message_id: messageId,
+      chat_id: chatId,
+      chat_type: flag(flags, 'chat-type') ?? 'group',
+      message_type: 'text',
+      content: JSON.stringify({ text }),
+      ...(mentionOpenId ? { mentions: [{ id: { open_id: mentionOpenId }, name: flag(flags, 'mention-name') ?? 'Probe', key: '@_probe_1' }] } : {}),
+    },
+  };
+  const result = await serverClient.probeLarkEvent({ appId, envelope: 'im.message.receive_v1', data });
+  if (boolFlag(flags, 'json')) {
+    printJson(result);
+    return;
+  }
+  console.log(`Lark probe injected. app=${result.app_id} event=${result.raw_event_id} inserted=${result.inserted} handled=${result.handled} parse_ok=${result.parse_ok}`);
+}
+
+export async function runMessagingCommand(serverClient: LockClient, args: MessagingCliArgs, usage: () => string): Promise<void> {
+  const sub = args.values[0];
+
+  if (sub === 'status') {
+    await printMessagingStatus(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'health') {
+    await printMessagingHealth(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'restart-lark') {
+    const appId = flag(args.flags, 'app-id') ?? flag(args.flags, 'id');
+    const result = await serverClient.restartLarkWebSocket(appId ? { appId } : {});
+    if (args.flags.json) {
+      printJson(result);
+    } else {
+      console.log(`Lark websocket restart requested. restarted=${result.restarted.join(',') || '-'} missing=${result.missing.join(',') || '-'}`);
+    }
+    return;
+  }
+
+  if (sub === 'repair-lark') {
+    await repairLarkMessaging(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'doctor-lark') {
+    await doctorLarkMessaging(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'watch-lark') {
+    await watchLarkMessaging(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'verify-lark-ingress') {
+    await verifyLarkIngressMessaging(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'recent-lark') {
+    await printRecentLarkEvents(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'repair-lark-events') {
+    await repairLarkEventParseFailures(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'probe-delivery') {
+    await probeDeliveryMessaging(serverClient, args.flags);
+    return;
+  }
+
+  if (sub === 'probe-lark') {
+    await probeLarkMessaging(serverClient, args.flags);
+    return;
+  }
+
+  throw new Error(`unknown messaging subcommand: ${sub ?? '(none)'}\n\n${usage()}`);
+}