@swarmclawai/swarmclaw 0.5.3 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. package/README.md +53 -9
  2. package/bin/server-cmd.js +1 -0
  3. package/bin/swarmclaw.js +76 -16
  4. package/next.config.ts +11 -1
  5. package/package.json +5 -2
  6. package/scripts/postinstall.mjs +18 -0
  7. package/src/app/api/canvas/[sessionId]/route.ts +31 -0
  8. package/src/app/api/chatrooms/[id]/chat/route.ts +284 -0
  9. package/src/app/api/chatrooms/[id]/members/route.ts +82 -0
  10. package/src/app/api/chatrooms/[id]/pins/route.ts +39 -0
  11. package/src/app/api/chatrooms/[id]/reactions/route.ts +42 -0
  12. package/src/app/api/chatrooms/[id]/route.ts +84 -0
  13. package/src/app/api/chatrooms/route.ts +50 -0
  14. package/src/app/api/connectors/[id]/route.ts +1 -0
  15. package/src/app/api/connectors/route.ts +2 -1
  16. package/src/app/api/credentials/route.ts +2 -3
  17. package/src/app/api/files/open/route.ts +43 -0
  18. package/src/app/api/knowledge/[id]/route.ts +13 -2
  19. package/src/app/api/knowledge/route.ts +8 -1
  20. package/src/app/api/memory/route.ts +8 -0
  21. package/src/app/api/notifications/route.ts +4 -0
  22. package/src/app/api/orchestrator/run/route.ts +1 -1
  23. package/src/app/api/plugins/install/route.ts +2 -2
  24. package/src/app/api/search/route.ts +53 -1
  25. package/src/app/api/sessions/[id]/chat/route.ts +2 -0
  26. package/src/app/api/sessions/[id]/edit-resend/route.ts +1 -1
  27. package/src/app/api/sessions/[id]/fork/route.ts +1 -1
  28. package/src/app/api/sessions/[id]/messages/route.ts +70 -2
  29. package/src/app/api/sessions/[id]/route.ts +4 -0
  30. package/src/app/api/sessions/route.ts +3 -3
  31. package/src/app/api/settings/route.ts +9 -0
  32. package/src/app/api/setup/check-provider/route.ts +3 -16
  33. package/src/app/api/skills/[id]/route.ts +6 -0
  34. package/src/app/api/skills/route.ts +6 -0
  35. package/src/app/api/tasks/[id]/route.ts +12 -0
  36. package/src/app/api/tasks/bulk/route.ts +100 -0
  37. package/src/app/api/tasks/metrics/route.ts +101 -0
  38. package/src/app/api/tasks/route.ts +18 -2
  39. package/src/app/api/tts/route.ts +3 -2
  40. package/src/app/api/tts/stream/route.ts +3 -2
  41. package/src/app/api/uploads/[filename]/route.ts +19 -34
  42. package/src/app/api/uploads/route.ts +94 -0
  43. package/src/app/api/webhooks/[id]/route.ts +15 -1
  44. package/src/app/globals.css +63 -15
  45. package/src/app/page.tsx +142 -13
  46. package/src/cli/index.js +40 -1
  47. package/src/cli/index.test.js +30 -0
  48. package/src/cli/spec.js +42 -0
  49. package/src/components/agents/agent-avatar.tsx +57 -10
  50. package/src/components/agents/agent-card.tsx +50 -17
  51. package/src/components/agents/agent-chat-list.tsx +148 -12
  52. package/src/components/agents/agent-list.tsx +50 -19
  53. package/src/components/agents/agent-sheet.tsx +120 -65
  54. package/src/components/agents/inspector-panel.tsx +81 -6
  55. package/src/components/agents/openclaw-skills-panel.tsx +32 -3
  56. package/src/components/agents/personality-builder.tsx +42 -14
  57. package/src/components/agents/soul-library-picker.tsx +89 -0
  58. package/src/components/auth/access-key-gate.tsx +10 -3
  59. package/src/components/auth/setup-wizard.tsx +2 -2
  60. package/src/components/auth/user-picker.tsx +31 -3
  61. package/src/components/canvas/canvas-panel.tsx +96 -0
  62. package/src/components/chat/activity-moment.tsx +173 -0
  63. package/src/components/chat/chat-area.tsx +46 -22
  64. package/src/components/chat/chat-header.tsx +457 -286
  65. package/src/components/chat/chat-preview-panel.tsx +1 -2
  66. package/src/components/chat/chat-tool-toggles.tsx +1 -1
  67. package/src/components/chat/delegation-banner.tsx +371 -0
  68. package/src/components/chat/file-path-chip.tsx +146 -0
  69. package/src/components/chat/heartbeat-history-panel.tsx +269 -0
  70. package/src/components/chat/markdown-utils.ts +9 -0
  71. package/src/components/chat/message-bubble.tsx +356 -315
  72. package/src/components/chat/message-list.tsx +230 -8
  73. package/src/components/chat/streaming-bubble.tsx +104 -47
  74. package/src/components/chat/suggestions-bar.tsx +1 -1
  75. package/src/components/chat/thinking-indicator.tsx +72 -10
  76. package/src/components/chat/tool-call-bubble.tsx +111 -73
  77. package/src/components/chat/tool-request-banner.tsx +31 -7
  78. package/src/components/chat/transfer-agent-picker.tsx +63 -0
  79. package/src/components/chatrooms/agent-hover-card.tsx +124 -0
  80. package/src/components/chatrooms/chatroom-input.tsx +320 -0
  81. package/src/components/chatrooms/chatroom-list.tsx +130 -0
  82. package/src/components/chatrooms/chatroom-message.tsx +432 -0
  83. package/src/components/chatrooms/chatroom-sheet.tsx +215 -0
  84. package/src/components/chatrooms/chatroom-tool-request-banner.tsx +134 -0
  85. package/src/components/chatrooms/chatroom-typing-bar.tsx +88 -0
  86. package/src/components/chatrooms/chatroom-view.tsx +344 -0
  87. package/src/components/chatrooms/reaction-picker.tsx +273 -0
  88. package/src/components/connectors/connector-list.tsx +168 -90
  89. package/src/components/connectors/connector-sheet.tsx +95 -56
  90. package/src/components/home/home-view.tsx +501 -0
  91. package/src/components/input/chat-input.tsx +107 -43
  92. package/src/components/knowledge/knowledge-list.tsx +31 -1
  93. package/src/components/knowledge/knowledge-sheet.tsx +83 -2
  94. package/src/components/layout/app-layout.tsx +194 -97
  95. package/src/components/layout/update-banner.tsx +2 -2
  96. package/src/components/logs/log-list.tsx +2 -2
  97. package/src/components/mcp-servers/mcp-server-sheet.tsx +1 -1
  98. package/src/components/memory/memory-agent-list.tsx +143 -0
  99. package/src/components/memory/memory-browser.tsx +205 -0
  100. package/src/components/memory/memory-card.tsx +34 -7
  101. package/src/components/memory/memory-detail.tsx +359 -120
  102. package/src/components/memory/memory-sheet.tsx +157 -23
  103. package/src/components/plugins/plugin-list.tsx +1 -1
  104. package/src/components/plugins/plugin-sheet.tsx +1 -1
  105. package/src/components/projects/project-detail.tsx +509 -0
  106. package/src/components/projects/project-list.tsx +195 -59
  107. package/src/components/providers/provider-list.tsx +2 -2
  108. package/src/components/providers/provider-sheet.tsx +3 -3
  109. package/src/components/schedules/schedule-card.tsx +1 -1
  110. package/src/components/schedules/schedule-list.tsx +1 -1
  111. package/src/components/schedules/schedule-sheet.tsx +259 -126
  112. package/src/components/secrets/secret-sheet.tsx +47 -24
  113. package/src/components/secrets/secrets-list.tsx +18 -8
  114. package/src/components/sessions/new-session-sheet.tsx +33 -65
  115. package/src/components/sessions/session-card.tsx +45 -14
  116. package/src/components/sessions/session-list.tsx +35 -18
  117. package/src/components/settings/gateway-disconnect-overlay.tsx +80 -0
  118. package/src/components/shared/agent-picker-list.tsx +90 -0
  119. package/src/components/shared/agent-switch-dialog.tsx +156 -0
  120. package/src/components/shared/attachment-chip.tsx +165 -0
  121. package/src/components/shared/avatar.tsx +10 -1
  122. package/src/components/shared/chatroom-picker-list.tsx +61 -0
  123. package/src/components/shared/check-icon.tsx +12 -0
  124. package/src/components/shared/confirm-dialog.tsx +1 -1
  125. package/src/components/shared/connector-platform-icon.tsx +51 -4
  126. package/src/components/shared/empty-state.tsx +32 -0
  127. package/src/components/shared/file-preview.tsx +34 -0
  128. package/src/components/shared/form-styles.ts +2 -0
  129. package/src/components/shared/icon-button.tsx +16 -2
  130. package/src/components/shared/keyboard-shortcuts-dialog.tsx +116 -0
  131. package/src/components/shared/notification-center.tsx +44 -6
  132. package/src/components/shared/profile-sheet.tsx +115 -0
  133. package/src/components/shared/reply-quote.tsx +26 -0
  134. package/src/components/shared/search-dialog.tsx +31 -15
  135. package/src/components/shared/section-label.tsx +12 -0
  136. package/src/components/shared/settings/plugin-manager.tsx +1 -1
  137. package/src/components/shared/settings/section-embedding.tsx +48 -13
  138. package/src/components/shared/settings/section-orchestrator.tsx +46 -15
  139. package/src/components/shared/settings/section-providers.tsx +1 -1
  140. package/src/components/shared/settings/section-secrets.tsx +1 -1
  141. package/src/components/shared/settings/section-storage.tsx +206 -0
  142. package/src/components/shared/settings/section-theme.tsx +95 -0
  143. package/src/components/shared/settings/section-user-preferences.tsx +57 -0
  144. package/src/components/shared/settings/section-voice.tsx +42 -21
  145. package/src/components/shared/settings/section-web-search.tsx +30 -6
  146. package/src/components/shared/settings/settings-page.tsx +182 -27
  147. package/src/components/shared/settings/settings-sheet.tsx +9 -73
  148. package/src/components/shared/settings/storage-browser.tsx +259 -0
  149. package/src/components/shared/sheet-footer.tsx +33 -0
  150. package/src/components/skills/skill-list.tsx +61 -30
  151. package/src/components/skills/skill-sheet.tsx +81 -2
  152. package/src/components/tasks/task-board.tsx +448 -26
  153. package/src/components/tasks/task-card.tsx +59 -9
  154. package/src/components/tasks/task-column.tsx +62 -3
  155. package/src/components/tasks/task-list.tsx +12 -4
  156. package/src/components/tasks/task-sheet.tsx +416 -74
  157. package/src/components/ui/hover-card.tsx +52 -0
  158. package/src/components/usage/metrics-dashboard.tsx +90 -6
  159. package/src/components/usage/usage-list.tsx +1 -1
  160. package/src/components/webhooks/webhook-sheet.tsx +1 -1
  161. package/src/hooks/use-continuous-speech.ts +10 -4
  162. package/src/hooks/use-view-router.ts +69 -19
  163. package/src/hooks/use-voice-conversation.ts +53 -10
  164. package/src/hooks/use-ws.ts +4 -2
  165. package/src/instrumentation.ts +15 -1
  166. package/src/lib/chat.ts +2 -0
  167. package/src/lib/memory.ts +3 -0
  168. package/src/lib/providers/anthropic.ts +13 -7
  169. package/src/lib/providers/index.ts +1 -0
  170. package/src/lib/providers/openai.ts +13 -7
  171. package/src/lib/server/chat-execution.ts +75 -15
  172. package/src/lib/server/chatroom-helpers.ts +146 -0
  173. package/src/lib/server/connectors/manager.ts +229 -7
  174. package/src/lib/server/context-manager.ts +225 -13
  175. package/src/lib/server/create-notification.ts +14 -2
  176. package/src/lib/server/daemon-state.ts +157 -10
  177. package/src/lib/server/execution-log.ts +1 -0
  178. package/src/lib/server/heartbeat-service.ts +48 -6
  179. package/src/lib/server/heartbeat-wake.ts +110 -0
  180. package/src/lib/server/langgraph-checkpoint.ts +1 -0
  181. package/src/lib/server/main-agent-loop.ts +1 -1
  182. package/src/lib/server/memory-consolidation.ts +105 -0
  183. package/src/lib/server/memory-db.ts +183 -10
  184. package/src/lib/server/mime.ts +51 -0
  185. package/src/lib/server/openclaw-gateway.ts +9 -1
  186. package/src/lib/server/orchestrator-lg.ts +2 -0
  187. package/src/lib/server/orchestrator.ts +5 -2
  188. package/src/lib/server/playwright-proxy.mjs +2 -3
  189. package/src/lib/server/prompt-runtime-context.ts +53 -0
  190. package/src/lib/server/provider-health.ts +125 -0
  191. package/src/lib/server/queue.ts +56 -10
  192. package/src/lib/server/scheduler.ts +8 -0
  193. package/src/lib/server/session-run-manager.ts +4 -0
  194. package/src/lib/server/session-tools/canvas.ts +67 -0
  195. package/src/lib/server/session-tools/chatroom.ts +136 -0
  196. package/src/lib/server/session-tools/connector.ts +83 -9
  197. package/src/lib/server/session-tools/context-mgmt.ts +36 -18
  198. package/src/lib/server/session-tools/crud.ts +21 -0
  199. package/src/lib/server/session-tools/delegate.ts +68 -4
  200. package/src/lib/server/session-tools/git.ts +71 -0
  201. package/src/lib/server/session-tools/http.ts +57 -0
  202. package/src/lib/server/session-tools/index.ts +10 -0
  203. package/src/lib/server/session-tools/memory.ts +7 -1
  204. package/src/lib/server/session-tools/search-providers.ts +16 -8
  205. package/src/lib/server/session-tools/subagent.ts +106 -0
  206. package/src/lib/server/session-tools/web.ts +115 -4
  207. package/src/lib/server/storage.ts +53 -29
  208. package/src/lib/server/stream-agent-chat.ts +185 -57
  209. package/src/lib/server/system-events.ts +49 -0
  210. package/src/lib/server/task-mention.ts +41 -0
  211. package/src/lib/server/ws-hub.ts +11 -0
  212. package/src/lib/sessions.ts +10 -0
  213. package/src/lib/soul-library.ts +103 -0
  214. package/src/lib/soul-suggestions.ts +109 -0
  215. package/src/lib/task-dedupe.ts +26 -0
  216. package/src/lib/tasks.ts +4 -1
  217. package/src/lib/tool-definitions.ts +2 -0
  218. package/src/lib/tts.ts +2 -2
  219. package/src/lib/view-routes.ts +36 -1
  220. package/src/lib/ws-client.ts +14 -4
  221. package/src/stores/use-app-store.ts +41 -3
  222. package/src/stores/use-chat-store.ts +113 -5
  223. package/src/stores/use-chatroom-store.ts +276 -0
  224. package/src/types/index.ts +88 -4
@@ -2,12 +2,27 @@ import { genId } from '@/lib/id'
2
2
  import {
3
3
  loadConnectors, saveConnectors, loadSessions, saveSessions,
4
4
  loadAgents, loadCredentials, decryptKey, loadSettings, loadSkills,
5
+ loadChatrooms, saveChatrooms,
5
6
  } from '../storage'
6
7
  import { WORKSPACE_DIR } from '../data-dir'
8
+ import { UPLOAD_DIR } from '../storage'
9
+ import fs from 'fs'
10
+ import path from 'path'
7
11
  import { streamAgentChat } from '../stream-agent-chat'
8
12
  import { notify } from '../ws-hub'
9
13
  import { logExecution } from '../execution-log'
10
- import type { Connector } from '@/types'
14
+ import { enqueueSystemEvent } from '../system-events'
15
+ import { requestHeartbeatNow } from '../heartbeat-wake'
16
+ import { buildCurrentDateTimePromptContext } from '../prompt-runtime-context'
17
+ import {
18
+ parseMentions,
19
+ buildChatroomSystemPrompt,
20
+ buildSyntheticSession,
21
+ buildAgentSystemPromptForChatroom,
22
+ buildHistoryForAgent,
23
+ resolveApiKey as resolveApiKeyHelper,
24
+ } from '../chatroom-helpers'
25
+ import type { Connector, MessageSource, Chatroom, ChatroomMessage } from '@/types'
11
26
  import type { ConnectorInstance, InboundMessage, InboundMedia } from './types'
12
27
  import {
13
28
  addAllowedSender,
@@ -21,6 +36,30 @@ import {
21
36
  type PairingPolicy,
22
37
  } from './pairing'
23
38
 
39
+ /**
40
+ * Extract embedded media references from agent response text.
41
+ * Parses markdown image/link patterns like ![alt](/api/uploads/filename)
42
+ * and resolves them to actual file paths on disk.
43
+ */
44
+ function extractEmbeddedMedia(text: string): { cleanText: string; files: Array<{ path: string; alt: string }> } {
45
+ const files: Array<{ path: string; alt: string }> = []
46
+ // Match markdown images: ![alt](/api/uploads/filename)
47
+ const imgRegex = /!\[([^\]]*)\]\(\/api\/uploads\/([^)]+)\)/g
48
+ let match: RegExpExecArray | null
49
+ while ((match = imgRegex.exec(text)) !== null) {
50
+ const [, alt, filename] = match
51
+ const safeName = filename.replace(/[^a-zA-Z0-9._-]/g, '')
52
+ const filePath = path.join(UPLOAD_DIR, safeName)
53
+ if (fs.existsSync(filePath)) {
54
+ files.push({ path: filePath, alt: alt || '' })
55
+ }
56
+ }
57
+ if (files.length === 0) return { cleanText: text, files }
58
+ // Strip the image markdown from text — the files will be sent as separate media
59
+ const cleanText = text.replace(imgRegex, '').replace(/\n{3,}/g, '\n\n').trim()
60
+ return { cleanText, files }
61
+ }
62
+
24
63
  /** Sentinel value agents return when no outbound reply should be sent */
25
64
  export const NO_MESSAGE_SENTINEL = 'NO_MESSAGE'
26
65
 
@@ -419,6 +458,132 @@ async function handleConnectorCommand(params: {
419
458
  return 'Unknown command.'
420
459
  }
421
460
 
461
+ /** Route an inbound message to a chatroom — process mentioned agents and return concatenated responses */
462
+ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage): Promise<string> {
463
+ const chatroomId = connector.chatroomId
464
+ if (!chatroomId) return '[Error] No chatroom configured.'
465
+
466
+ const chatrooms = loadChatrooms()
467
+ const chatroom = chatrooms[chatroomId] as Chatroom | undefined
468
+ if (!chatroom) return '[Error] Chatroom not found.'
469
+
470
+ const agents = loadAgents()
471
+ const source: MessageSource = {
472
+ platform: connector.platform,
473
+ connectorId: connector.id,
474
+ connectorName: connector.name,
475
+ senderName: msg.senderName,
476
+ }
477
+
478
+ // Parse mentions from the message text
479
+ let mentions = parseMentions(msg.text || '', agents, chatroom.agentIds)
480
+ // Auto-address: if enabled and no explicit mentions, address all agents
481
+ if (chatroom.autoAddress && mentions.length === 0) {
482
+ mentions = [...chatroom.agentIds]
483
+ }
484
+
485
+ // Create and persist the user message in the chatroom
486
+ const userMessage: ChatroomMessage = {
487
+ id: genId(),
488
+ senderId: 'user',
489
+ senderName: msg.senderName || 'User',
490
+ role: 'user',
491
+ text: msg.text || '',
492
+ mentions,
493
+ reactions: [],
494
+ time: Date.now(),
495
+ source,
496
+ }
497
+ chatroom.messages.push(userMessage)
498
+ chatroom.updatedAt = Date.now()
499
+ chatrooms[chatroomId] = chatroom
500
+ saveChatrooms(chatrooms)
501
+ notify('chatrooms')
502
+ notify(`chatroom:${chatroomId}`)
503
+
504
+ // Process mentioned agents sequentially and collect responses
505
+ const responses: string[] = []
506
+ for (const agentId of mentions) {
507
+ const agent = agents[agentId]
508
+ if (!agent) continue
509
+
510
+ const apiKey = resolveApiKeyHelper(agent.credentialId)
511
+ const freshChatrooms = loadChatrooms()
512
+ const freshChatroom = freshChatrooms[chatroomId] as Chatroom
513
+
514
+ const syntheticSession = buildSyntheticSession(agent, chatroomId)
515
+ const agentSystemPrompt = buildAgentSystemPromptForChatroom(agent)
516
+ const chatroomContext = buildChatroomSystemPrompt(freshChatroom, agents, agent.id)
517
+ const fullSystemPrompt = [agentSystemPrompt, chatroomContext].filter(Boolean).join('\n\n')
518
+ const history = buildHistoryForAgent(freshChatroom, agent.id)
519
+
520
+ try {
521
+ const result = await streamAgentChat({
522
+ session: syntheticSession,
523
+ message: msg.text || '',
524
+ apiKey,
525
+ systemPrompt: fullSystemPrompt,
526
+ write: () => {},
527
+ history,
528
+ })
529
+
530
+ const responseText = result.finalResponse || result.fullText
531
+ if (responseText.trim() && !isNoMessage(responseText)) {
532
+ // Persist agent response to chatroom
533
+ const agentSource: MessageSource = {
534
+ platform: connector.platform,
535
+ connectorId: connector.id,
536
+ connectorName: connector.name,
537
+ }
538
+ const agentMessage: ChatroomMessage = {
539
+ id: genId(),
540
+ senderId: agent.id,
541
+ senderName: agent.name,
542
+ role: 'assistant',
543
+ text: responseText,
544
+ mentions: parseMentions(responseText, agents, freshChatroom.agentIds),
545
+ reactions: [],
546
+ time: Date.now(),
547
+ source: agentSource,
548
+ }
549
+ const latestChatrooms = loadChatrooms()
550
+ const latestChatroom = latestChatrooms[chatroomId] as Chatroom
551
+ latestChatroom.messages.push(agentMessage)
552
+ latestChatroom.updatedAt = Date.now()
553
+ latestChatrooms[chatroomId] = latestChatroom
554
+ saveChatrooms(latestChatrooms)
555
+ notify(`chatroom:${chatroomId}`)
556
+
557
+ responses.push(`[${agent.name}] ${responseText}`)
558
+ }
559
+ } catch (err: unknown) {
560
+ const errMsg = err instanceof Error ? err.message : String(err)
561
+ console.error(`[connector] Chatroom agent ${agent.name} error:`, errMsg)
562
+ }
563
+ }
564
+
565
+ if (responses.length === 0) return NO_MESSAGE_SENTINEL
566
+
567
+ const joined = responses.join('\n\n')
568
+ // Extract embedded media from agent responses and send them via connector
569
+ const extracted = extractEmbeddedMedia(joined)
570
+ if (extracted.files.length > 0) {
571
+ const inst = running.get(connector.id)
572
+ if (inst?.sendMessage) {
573
+ for (const file of extracted.files) {
574
+ try {
575
+ await inst.sendMessage(msg.channelId, '', { mediaPath: file.path, caption: file.alt || undefined })
576
+ console.log(`[connector] Sent chatroom media to ${msg.platform}: ${path.basename(file.path)}`)
577
+ } catch (err: unknown) {
578
+ console.error(`[connector] Failed to send chatroom media ${path.basename(file.path)}:`, err instanceof Error ? err.message : String(err))
579
+ }
580
+ }
581
+ }
582
+ return extracted.cleanText || '(no response)'
583
+ }
584
+ return joined
585
+ }
586
+
422
587
  /** Route an inbound message through the assigned agent and return the response */
423
588
  async function routeMessage(connector: Connector, msg: InboundMessage): Promise<string> {
424
589
  if (msg?.channelId) {
@@ -426,11 +591,26 @@ async function routeMessage(connector: Connector, msg: InboundMessage): Promise<
426
591
  }
427
592
  lastInboundTimeByConnector.set(connector.id, Date.now())
428
593
 
594
+ // Route to chatroom if configured
595
+ if (connector.chatroomId) {
596
+ return routeMessageToChatroom(connector, msg)
597
+ }
598
+
429
599
  const agents = loadAgents()
430
600
  const effectiveAgentId = msg.agentIdOverride || connector.agentId
601
+ if (!effectiveAgentId) return '[Error] Connector has no agent configured.'
431
602
  const agent = agents[effectiveAgentId]
432
603
  if (!agent) return '[Error] Connector agent not found.'
433
604
 
605
+ // Enqueue system event + heartbeat wake for the agent
606
+ const preview = (msg.text || '').slice(0, 80)
607
+ enqueueSystemEvent(
608
+ `connector:${connector.id}:${msg.channelId}`,
609
+ `Inbound message from ${msg.platform}: ${preview}`,
610
+ 'connector-message',
611
+ )
612
+ requestHeartbeatNow({ agentId: effectiveAgentId, reason: 'connector-message' })
613
+
434
614
  // Log connector trigger
435
615
  const triggerSessionKey = `connector:${connector.id}:${msg.channelId}`
436
616
  const allSessions = loadSessions()
@@ -461,11 +641,16 @@ async function routeMessage(connector: Connector, msg: InboundMessage): Promise<
461
641
  }
462
642
  }
463
643
 
464
- // Find or create a session keyed by platform + channel
644
+ // Find a session for this connector message.
645
+ // Prefer the agent's thread session (visible in the agent chat UI) so connector
646
+ // messages appear inline alongside web UI messages.
647
+ // Fall back to a connector-keyed session if the agent has no thread session.
465
648
  const sessionKey = `connector:${connector.id}:${msg.channelId}`
466
649
  const sessions = loadSessions()
467
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
468
- let session = Object.values(sessions).find((s: any) => s.name === sessionKey)
650
+ let session = (agent.threadSessionId && sessions[agent.threadSessionId])
651
+ ? sessions[agent.threadSessionId]
652
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
653
+ : Object.values(sessions).find((s: any) => s.name === sessionKey)
469
654
  if (!session) {
470
655
  const id = genId()
471
656
  session = {
@@ -553,6 +738,7 @@ async function routeMessage(connector: Connector, msg: InboundMessage): Promise<
553
738
  const settings = loadSettings()
554
739
  const promptParts: string[] = []
555
740
  if (settings.userPrompt) promptParts.push(settings.userPrompt)
741
+ promptParts.push(buildCurrentDateTimePromptContext())
556
742
  if (agent.soul) promptParts.push(agent.soul)
557
743
  if (agent.systemPrompt) promptParts.push(agent.systemPrompt)
558
744
  if (agent.skillIds?.length) {
@@ -583,12 +769,22 @@ The test: would a thoughtful friend feel compelled to type something back? If no
583
769
  const firstImageUrl = msg.imageUrl || (firstImage?.url) || undefined
584
770
  const firstImagePath = firstImage?.localPath || undefined
585
771
  const inboundText = formatInboundUserText(msg)
772
+ // Store the raw user text for display (source.senderName handles attribution).
773
+ // The formatted text with [SenderName] prefix is only used for LLM history context.
774
+ const rawText = (msg.text || '').trim()
775
+ const messageSource: MessageSource = {
776
+ platform: connector.platform,
777
+ connectorId: connector.id,
778
+ connectorName: connector.name,
779
+ senderName: msg.senderName,
780
+ }
586
781
  session.messages.push({
587
782
  role: 'user',
588
- text: inboundText,
783
+ text: rawText || inboundText,
589
784
  time: Date.now(),
590
785
  imageUrl: firstImageUrl,
591
786
  imagePath: firstImagePath,
787
+ source: messageSource,
592
788
  })
593
789
  session.lastActiveAt = Date.now()
594
790
  const s1 = loadSessions()
@@ -668,9 +864,14 @@ The test: would a thoughtful friend feel compelled to type something back? If no
668
864
  },
669
865
  })
670
866
 
671
- // Save assistant response to session
867
+ // Save assistant response to session (full text with image markdown for web UI rendering)
868
+ const assistantSource: MessageSource = {
869
+ platform: connector.platform,
870
+ connectorId: connector.id,
871
+ connectorName: connector.name,
872
+ }
672
873
  if (fullText.trim()) {
673
- session.messages.push({ role: 'assistant', text: fullText.trim(), time: Date.now() })
874
+ session.messages.push({ role: 'assistant', text: fullText.trim(), time: Date.now(), source: assistantSource })
674
875
  session.lastActiveAt = Date.now()
675
876
  const s2 = loadSessions()
676
877
  s2[session.id] = session
@@ -678,6 +879,24 @@ The test: would a thoughtful friend feel compelled to type something back? If no
678
879
  notify(`messages:${session.id}`)
679
880
  }
680
881
 
882
+ // Extract embedded media (screenshots, uploaded files) and send them as separate
883
+ // media messages via the connector, then return the cleaned text
884
+ const extracted = extractEmbeddedMedia(fullText)
885
+ if (extracted.files.length > 0) {
886
+ const inst = running.get(connector.id)
887
+ if (inst?.sendMessage) {
888
+ for (const file of extracted.files) {
889
+ try {
890
+ await inst.sendMessage(msg.channelId, '', { mediaPath: file.path, caption: file.alt || undefined })
891
+ console.log(`[connector] Sent media to ${msg.platform}: ${path.basename(file.path)}`)
892
+ } catch (err: unknown) {
893
+ console.error(`[connector] Failed to send media ${path.basename(file.path)}:`, err instanceof Error ? err.message : String(err))
894
+ }
895
+ }
896
+ }
897
+ return extracted.cleanText || '(no response)'
898
+ }
899
+
681
900
  return fullText || '(no response)'
682
901
  }
683
902
 
@@ -862,6 +1081,7 @@ export function listRunningConnectors(platform?: string): Array<{
862
1081
  id: string
863
1082
  name: string
864
1083
  platform: string
1084
+ agentId: string | null
865
1085
  supportsSend: boolean
866
1086
  configuredTargets: string[]
867
1087
  recentChannelId: string | null
@@ -871,6 +1091,7 @@ export function listRunningConnectors(platform?: string): Array<{
871
1091
  id: string
872
1092
  name: string
873
1093
  platform: string
1094
+ agentId: string | null
874
1095
  supportsSend: boolean
875
1096
  configuredTargets: string[]
876
1097
  recentChannelId: string | null
@@ -896,6 +1117,7 @@ export function listRunningConnectors(platform?: string): Array<{
896
1117
  id,
897
1118
  name: connector.name,
898
1119
  platform: connector.platform,
1120
+ agentId: connector.agentId || null,
899
1121
  supportsSend: typeof instance.sendMessage === 'function',
900
1122
  configuredTargets: Array.from(new Set(configuredTargets)),
901
1123
  recentChannelId: lastInboundChannelByConnector.get(id) || null,
@@ -1,6 +1,17 @@
1
- import type { Message, ProviderType } from '@/types'
1
+ import type { Message } from '@/types'
2
2
  import { getMemoryDb } from './memory-db'
3
3
 
4
+ // --- LLM compaction constants ---
5
+
6
+ const COMPACTION_CHUNK_BUDGET_RATIO = 0.4 // 40% of context per summarization chunk
7
+ const COMPACTION_SAFETY_MARGIN = 1.2 // 20% buffer for token underestimation
8
+ const COMPACTION_OVERHEAD_TOKENS = 4096 // reserved for summarization prompt + response
9
+ const MAX_TOOL_FAILURES = 8
10
+ const MAX_FAILURE_CHARS = 240
11
+
12
+ /** Callback that sends a prompt to an LLM and returns response text */
13
+ export type LLMSummarizer = (prompt: string) => Promise<string>
14
+
4
15
  // --- Context window sizes (tokens) per provider/model ---
5
16
 
6
17
  const PROVIDER_CONTEXT_WINDOWS: Record<string, number> = {
@@ -160,6 +171,125 @@ export function consolidateToMemory(
160
171
  return stored
161
172
  }
162
173
 
174
+ // --- LLM compaction helpers ---
175
+
176
+ /** Extract recent tool failures from messages for metadata appendix */
177
+ export function extractToolFailures(messages: Message[]): string[] {
178
+ const failures: string[] = []
179
+ for (const m of messages) {
180
+ if (!m.toolEvents) continue
181
+ for (const te of m.toolEvents) {
182
+ if (!te.error) continue
183
+ const snippet = (te.output || '').slice(0, MAX_FAILURE_CHARS)
184
+ failures.push(`[${te.name}] error: ${snippet}`)
185
+ }
186
+ }
187
+ return failures.slice(-MAX_TOOL_FAILURES)
188
+ }
189
+
190
+ /** Extract file paths read and modified from tool events */
191
+ export function extractFileOperations(messages: Message[]): { read: string[]; modified: string[] } {
192
+ const readSet = new Set<string>()
193
+ const modifiedSet = new Set<string>()
194
+
195
+ const READ_TOOLS = new Set(['read_file', 'list_files'])
196
+ const WRITE_TOOLS = new Set(['write_file', 'edit_file', 'copy_file', 'move_file', 'delete_file'])
197
+
198
+ for (const m of messages) {
199
+ if (!m.toolEvents) continue
200
+ for (const te of m.toolEvents) {
201
+ let parsed: Record<string, unknown> | null = null
202
+ try { parsed = JSON.parse(te.input) } catch { /* not JSON */ }
203
+ if (!parsed) continue
204
+
205
+ const paths: string[] = []
206
+ for (const key of ['filePath', 'sourcePath', 'destinationPath']) {
207
+ const v = parsed[key]
208
+ if (typeof v === 'string' && v) paths.push(v)
209
+ }
210
+
211
+ const isRead = READ_TOOLS.has(te.name)
212
+ const isWrite = WRITE_TOOLS.has(te.name)
213
+ for (const p of paths) {
214
+ if (isWrite) modifiedSet.add(p)
215
+ else if (isRead) readSet.add(p)
216
+ }
217
+ }
218
+ }
219
+ return { read: [...readSet], modified: [...modifiedSet] }
220
+ }
221
+
222
+ /** Split messages into chunks that fit within a token budget each */
223
+ export function splitMessagesByTokenBudget(messages: Message[], budgetPerChunk: number): Message[][] {
224
+ if (messages.length === 0) return []
225
+ const chunks: Message[][] = []
226
+ let current: Message[] = []
227
+ let currentTokens = 0
228
+
229
+ for (const m of messages) {
230
+ const msgTokens = estimateMessagesTokens([m])
231
+ if (current.length > 0 && currentTokens + msgTokens > budgetPerChunk) {
232
+ chunks.push(current)
233
+ current = []
234
+ currentTokens = 0
235
+ }
236
+ current.push(m)
237
+ currentTokens += msgTokens
238
+ }
239
+ if (current.length > 0) chunks.push(current)
240
+ return chunks
241
+ }
242
+
243
+ /** Build an OpenClaw-aligned summarization prompt for a batch of messages */
244
+ function buildSummarizationPrompt(messages: Message[]): string {
245
+ const transcript = messages.map((m) => {
246
+ let line = `[${m.role}]: ${m.text}`
247
+ if (m.toolEvents?.length) {
248
+ for (const te of m.toolEvents) {
249
+ const inp = (te.input || '').slice(0, 500)
250
+ const out = (te.output || '').slice(0, 500)
251
+ line += `\n tool:${te.name}(${inp})${te.error ? ' [ERROR]' : ''} → ${out}`
252
+ }
253
+ }
254
+ return line
255
+ }).join('\n\n')
256
+
257
+ return [
258
+ 'Summarize the following conversation transcript into structured notes.',
259
+ '',
260
+ 'Rules:',
261
+ '- Preserve all decisions, TODOs, open questions, and constraints',
262
+ '- Preserve all opaque identifiers exactly as they appear (UUIDs, hashes, IDs, URLs, file paths, API keys, variable names)',
263
+ '- Note errors encountered and their resolutions',
264
+ '- Keep technical details needed to continue work (versions, configs, commands)',
265
+ '- Aim for 20-40% of original length',
266
+ '- Use structured notes with bullet points, not narrative prose',
267
+ '- Group by topic/theme when possible',
268
+ '',
269
+ '---TRANSCRIPT---',
270
+ transcript,
271
+ '---END TRANSCRIPT---',
272
+ ].join('\n')
273
+ }
274
+
275
+ /** Build a merge prompt for combining multiple partial summaries */
276
+ function buildMergePrompt(partialSummaries: string[]): string {
277
+ const numbered = partialSummaries.map((s, i) => `--- Part ${i + 1} ---\n${s}`).join('\n\n')
278
+
279
+ return [
280
+ 'Merge the following partial conversation summaries into a single cohesive summary.',
281
+ '',
282
+ 'Rules:',
283
+ '- Remove redundancy across parts while preserving all important details',
284
+ '- Preserve all opaque identifiers exactly (UUIDs, hashes, IDs, URLs, file paths)',
285
+ '- Keep decisions, TODOs, open questions, constraints, and error resolutions',
286
+ '- Use structured notes with bullet points',
287
+ '- The result should be shorter than the combined input',
288
+ '',
289
+ numbered,
290
+ ].join('\n')
291
+ }
292
+
163
293
  // --- Compaction strategies ---
164
294
 
165
295
  export interface CompactionResult {
@@ -178,15 +308,18 @@ export function slidingWindowCompact(
178
308
  return messages.slice(-keepLastN)
179
309
  }
180
310
 
181
- /** Summarize old messages, keep recent ones */
182
- export async function summarizeAndCompact(opts: {
311
+ /** LLM-powered compaction: summarize old messages using an LLM, with progressive fallback */
312
+ export async function llmCompact(opts: {
183
313
  messages: Message[]
184
- keepLastN: number
314
+ provider: string
315
+ model: string
185
316
  agentId: string | null
186
317
  sessionId: string
187
- generateSummary: (text: string) => Promise<string>
318
+ summarize: LLMSummarizer
319
+ keepLastN?: number
188
320
  }): Promise<CompactionResult> {
189
- const { messages, keepLastN, agentId, sessionId, generateSummary } = opts
321
+ const { messages, provider, model, agentId, sessionId, summarize, keepLastN = 10 } = opts
322
+
190
323
  if (messages.length <= keepLastN) {
191
324
  return { messages, prunedCount: 0, memoriesStored: 0, summaryAdded: false }
192
325
  }
@@ -194,19 +327,75 @@ export async function summarizeAndCompact(opts: {
194
327
  const oldMessages = messages.slice(0, -keepLastN)
195
328
  const recentMessages = messages.slice(-keepLastN)
196
329
 
197
- // Consolidate important info to memory before pruning
330
+ // 1. Consolidate important info to memory (existing regex extraction)
198
331
  const memoriesStored = consolidateToMemory(oldMessages, agentId, sessionId)
199
332
 
200
- // Build text for summarization
201
- const conversationText = oldMessages
202
- .map((m) => `${m.role}: ${m.text}`)
203
- .join('\n\n')
333
+ // 2. Extract metadata from old messages
334
+ const toolFailures = extractToolFailures(oldMessages)
335
+ const fileOps = extractFileOperations(oldMessages)
336
+
337
+ // 3. Compute chunk budget
338
+ const contextWindow = getContextWindowSize(provider, model)
339
+ const chunkBudget = Math.floor((contextWindow / COMPACTION_SAFETY_MARGIN) * COMPACTION_CHUNK_BUDGET_RATIO) - COMPACTION_OVERHEAD_TOKENS
340
+
341
+ // 4. Split old messages into chunks
342
+ const chunks = splitMessagesByTokenBudget(oldMessages, Math.max(chunkBudget, 2000))
343
+
344
+ // 5. Summarize chunks (progressive fallback on failure)
345
+ let finalSummary: string | null = null
346
+ try {
347
+ if (chunks.length === 1) {
348
+ finalSummary = await summarize(buildSummarizationPrompt(chunks[0]))
349
+ } else {
350
+ // Multi-chunk: summarize each, then merge
351
+ const partialSummaries: string[] = []
352
+ for (const chunk of chunks) {
353
+ try {
354
+ const partial = await summarize(buildSummarizationPrompt(chunk))
355
+ if (partial?.trim()) partialSummaries.push(partial.trim())
356
+ } catch {
357
+ // Skip failed chunks — progressive fallback
358
+ }
359
+ }
360
+ if (partialSummaries.length === 0) {
361
+ finalSummary = null // all chunks failed
362
+ } else if (partialSummaries.length === 1) {
363
+ finalSummary = partialSummaries[0]
364
+ } else {
365
+ finalSummary = await summarize(buildMergePrompt(partialSummaries))
366
+ }
367
+ }
368
+ } catch {
369
+ finalSummary = null
370
+ }
371
+
372
+ // 6. Fall back to sliding window if LLM summarization failed entirely
373
+ if (!finalSummary?.trim()) {
374
+ return {
375
+ messages: slidingWindowCompact(messages, keepLastN),
376
+ prunedCount: oldMessages.length,
377
+ memoriesStored,
378
+ summaryAdded: false,
379
+ }
380
+ }
381
+
382
+ // 7. Append metadata sections
383
+ const metaSections: string[] = [finalSummary.trim()]
204
384
 
205
- const summary = await generateSummary(conversationText)
385
+ if (toolFailures.length > 0) {
386
+ metaSections.push('\n## Tool Failures\n' + toolFailures.join('\n'))
387
+ }
388
+ if (fileOps.read.length > 0 || fileOps.modified.length > 0) {
389
+ const parts: string[] = []
390
+ if (fileOps.read.length) parts.push('Read: ' + fileOps.read.join(', '))
391
+ if (fileOps.modified.length) parts.push('Modified: ' + fileOps.modified.join(', '))
392
+ metaSections.push('\n## File Operations\n' + parts.join('\n'))
393
+ }
206
394
 
395
+ // 8. Build context summary message
207
396
  const summaryMessage: Message = {
208
397
  role: 'assistant',
209
- text: `[Context Summary]\n${summary}`,
398
+ text: `[Context Summary]\n${metaSections.join('\n')}`,
210
399
  time: Date.now(),
211
400
  kind: 'system',
212
401
  }
@@ -219,6 +408,29 @@ export async function summarizeAndCompact(opts: {
219
408
  }
220
409
  }
221
410
 
411
+ /** Summarize old messages, keep recent ones. Delegates to llmCompact for LLM-powered summarization. */
412
+ export async function summarizeAndCompact(opts: {
413
+ messages: Message[]
414
+ keepLastN: number
415
+ agentId: string | null
416
+ sessionId: string
417
+ provider: string
418
+ model: string
419
+ generateSummary: LLMSummarizer
420
+ }): Promise<CompactionResult> {
421
+ const { messages, keepLastN, agentId, sessionId, provider, model, generateSummary } = opts
422
+
423
+ return llmCompact({
424
+ messages,
425
+ provider,
426
+ model,
427
+ agentId,
428
+ sessionId,
429
+ summarize: generateSummary,
430
+ keepLastN,
431
+ })
432
+ }
433
+
222
434
  /** Auto-compact: triggers when estimated tokens exceed threshold */
223
435
  export function shouldAutoCompact(
224
436
  messages: Message[],
@@ -1,26 +1,38 @@
1
1
  import { genId } from '@/lib/id'
2
- import { saveNotification } from '@/lib/server/storage'
2
+ import { saveNotification, hasUnreadNotificationWithKey } from '@/lib/server/storage'
3
3
  import { notify } from '@/lib/server/ws-hub'
4
4
  import type { AppNotification } from '@/types'
5
5
 
6
6
  /**
7
7
  * Create and persist a notification, then push a WS invalidation.
8
+ * If `dedupKey` is provided and an unread notification with the same key
9
+ * already exists, returns `null` (no insert, no WS push).
8
10
  */
9
11
  export function createNotification(opts: {
10
12
  type: AppNotification['type']
11
13
  title: string
12
14
  message?: string
15
+ actionLabel?: string
16
+ actionUrl?: string
13
17
  entityType?: string
14
18
  entityId?: string
15
- }) {
19
+ dedupKey?: string
20
+ }): AppNotification | null {
21
+ if (opts.dedupKey && hasUnreadNotificationWithKey(opts.dedupKey)) {
22
+ return null
23
+ }
24
+
16
25
  const id = genId()
17
26
  const notification: AppNotification = {
18
27
  id,
19
28
  type: opts.type,
20
29
  title: opts.title,
21
30
  message: opts.message,
31
+ actionLabel: opts.actionLabel,
32
+ actionUrl: opts.actionUrl,
22
33
  entityType: opts.entityType,
23
34
  entityId: opts.entityId,
35
+ dedupKey: opts.dedupKey,
24
36
  read: false,
25
37
  createdAt: Date.now(),
26
38
  }