@swarmclawai/swarmclaw 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +15 -2
  2. package/bin/server-cmd.js +1 -0
  3. package/package.json +2 -1
  4. package/src/app/api/canvas/[sessionId]/route.ts +31 -0
  5. package/src/app/api/chatrooms/[id]/chat/route.ts +10 -136
  6. package/src/app/api/connectors/[id]/route.ts +1 -0
  7. package/src/app/api/connectors/route.ts +2 -1
  8. package/src/app/api/files/open/route.ts +43 -0
  9. package/src/app/api/search/route.ts +9 -7
  10. package/src/app/api/sessions/[id]/messages/route.ts +70 -2
  11. package/src/app/api/sessions/[id]/route.ts +4 -0
  12. package/src/app/api/tasks/metrics/route.ts +101 -0
  13. package/src/app/api/tasks/route.ts +17 -2
  14. package/src/app/api/tts/route.ts +3 -2
  15. package/src/app/api/tts/stream/route.ts +3 -2
  16. package/src/app/api/uploads/[filename]/route.ts +19 -34
  17. package/src/app/api/uploads/route.ts +94 -0
  18. package/src/app/globals.css +5 -0
  19. package/src/cli/index.js +16 -1
  20. package/src/cli/spec.js +26 -0
  21. package/src/components/agents/agent-card.tsx +3 -3
  22. package/src/components/agents/agent-chat-list.tsx +29 -6
  23. package/src/components/agents/agent-sheet.tsx +66 -4
  24. package/src/components/agents/inspector-panel.tsx +81 -6
  25. package/src/components/agents/openclaw-skills-panel.tsx +32 -3
  26. package/src/components/agents/personality-builder.tsx +42 -14
  27. package/src/components/agents/soul-library-picker.tsx +89 -0
  28. package/src/components/canvas/canvas-panel.tsx +96 -0
  29. package/src/components/chat/activity-moment.tsx +8 -4
  30. package/src/components/chat/chat-area.tsx +46 -22
  31. package/src/components/chat/chat-header.tsx +455 -286
  32. package/src/components/chat/chat-preview-panel.tsx +1 -2
  33. package/src/components/chat/delegation-banner.tsx +371 -0
  34. package/src/components/chat/file-path-chip.tsx +23 -2
  35. package/src/components/chat/heartbeat-history-panel.tsx +269 -0
  36. package/src/components/chat/message-bubble.tsx +315 -25
  37. package/src/components/chat/message-list.tsx +180 -7
  38. package/src/components/chat/streaming-bubble.tsx +68 -1
  39. package/src/components/chat/tool-call-bubble.tsx +45 -3
  40. package/src/components/chat/transfer-agent-picker.tsx +1 -1
  41. package/src/components/chatrooms/chatroom-list.tsx +8 -1
  42. package/src/components/chatrooms/chatroom-message.tsx +8 -3
  43. package/src/components/chatrooms/chatroom-view.tsx +3 -3
  44. package/src/components/connectors/connector-list.tsx +168 -90
  45. package/src/components/connectors/connector-sheet.tsx +68 -16
  46. package/src/components/home/home-view.tsx +1 -1
  47. package/src/components/input/chat-input.tsx +28 -2
  48. package/src/components/layout/app-layout.tsx +19 -2
  49. package/src/components/projects/project-detail.tsx +1 -1
  50. package/src/components/schedules/schedule-sheet.tsx +260 -127
  51. package/src/components/settings/gateway-disconnect-overlay.tsx +80 -0
  52. package/src/components/shared/agent-switch-dialog.tsx +1 -1
  53. package/src/components/shared/chatroom-picker-list.tsx +61 -0
  54. package/src/components/shared/connector-platform-icon.tsx +51 -4
  55. package/src/components/shared/icon-button.tsx +16 -2
  56. package/src/components/shared/keyboard-shortcuts-dialog.tsx +1 -1
  57. package/src/components/shared/search-dialog.tsx +17 -10
  58. package/src/components/shared/settings/section-embedding.tsx +48 -13
  59. package/src/components/shared/settings/section-orchestrator.tsx +46 -15
  60. package/src/components/shared/settings/section-storage.tsx +206 -0
  61. package/src/components/shared/settings/section-user-preferences.tsx +18 -0
  62. package/src/components/shared/settings/section-voice.tsx +42 -21
  63. package/src/components/shared/settings/section-web-search.tsx +30 -6
  64. package/src/components/shared/settings/settings-page.tsx +3 -1
  65. package/src/components/shared/settings/storage-browser.tsx +259 -0
  66. package/src/components/tasks/task-card.tsx +14 -1
  67. package/src/components/tasks/task-sheet.tsx +328 -3
  68. package/src/components/usage/metrics-dashboard.tsx +90 -6
  69. package/src/hooks/use-continuous-speech.ts +10 -4
  70. package/src/hooks/use-voice-conversation.ts +53 -10
  71. package/src/hooks/use-ws.ts +4 -2
  72. package/src/lib/providers/anthropic.ts +13 -7
  73. package/src/lib/providers/index.ts +1 -0
  74. package/src/lib/providers/openai.ts +13 -7
  75. package/src/lib/server/chat-execution.ts +51 -11
  76. package/src/lib/server/chatroom-helpers.ts +146 -0
  77. package/src/lib/server/connectors/manager.ts +218 -7
  78. package/src/lib/server/heartbeat-service.ts +8 -1
  79. package/src/lib/server/main-agent-loop.ts +1 -1
  80. package/src/lib/server/memory-consolidation.ts +15 -2
  81. package/src/lib/server/memory-db.ts +134 -6
  82. package/src/lib/server/mime.ts +51 -0
  83. package/src/lib/server/openclaw-gateway.ts +2 -2
  84. package/src/lib/server/orchestrator-lg.ts +2 -0
  85. package/src/lib/server/orchestrator.ts +5 -2
  86. package/src/lib/server/playwright-proxy.mjs +2 -3
  87. package/src/lib/server/prompt-runtime-context.ts +53 -0
  88. package/src/lib/server/queue.ts +52 -7
  89. package/src/lib/server/session-tools/canvas.ts +67 -0
  90. package/src/lib/server/session-tools/connector.ts +83 -9
  91. package/src/lib/server/session-tools/crud.ts +21 -0
  92. package/src/lib/server/session-tools/delegate.ts +68 -4
  93. package/src/lib/server/session-tools/git.ts +71 -0
  94. package/src/lib/server/session-tools/http.ts +57 -0
  95. package/src/lib/server/session-tools/index.ts +8 -0
  96. package/src/lib/server/session-tools/memory.ts +1 -0
  97. package/src/lib/server/session-tools/search-providers.ts +16 -8
  98. package/src/lib/server/session-tools/subagent.ts +106 -0
  99. package/src/lib/server/session-tools/web.ts +115 -4
  100. package/src/lib/server/stream-agent-chat.ts +32 -10
  101. package/src/lib/server/task-mention.ts +41 -0
  102. package/src/lib/sessions.ts +10 -0
  103. package/src/lib/soul-library.ts +103 -0
  104. package/src/lib/task-dedupe.ts +26 -0
  105. package/src/lib/tool-definitions.ts +2 -0
  106. package/src/lib/tts.ts +2 -2
  107. package/src/stores/use-app-store.ts +5 -1
  108. package/src/stores/use-chat-store.ts +65 -2
  109. package/src/types/index.ts +32 -2
@@ -0,0 +1,51 @@
1
+ export const MIME_TYPES: Record<string, string> = {
2
+ '.png': 'image/png',
3
+ '.jpg': 'image/jpeg',
4
+ '.jpeg': 'image/jpeg',
5
+ '.gif': 'image/gif',
6
+ '.webp': 'image/webp',
7
+ '.svg': 'image/svg+xml',
8
+ '.bmp': 'image/bmp',
9
+ '.ico': 'image/x-icon',
10
+ '.mp4': 'video/mp4',
11
+ '.webm': 'video/webm',
12
+ '.mov': 'video/quicktime',
13
+ '.avi': 'video/x-msvideo',
14
+ '.mkv': 'video/x-matroska',
15
+ '.pdf': 'application/pdf',
16
+ '.json': 'application/json',
17
+ '.csv': 'text/csv',
18
+ '.txt': 'text/plain',
19
+ '.html': 'text/html',
20
+ '.xml': 'application/xml',
21
+ '.zip': 'application/zip',
22
+ '.tar': 'application/x-tar',
23
+ '.gz': 'application/gzip',
24
+ '.doc': 'application/msword',
25
+ '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
26
+ '.xls': 'application/vnd.ms-excel',
27
+ '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
28
+ '.ppt': 'application/vnd.ms-powerpoint',
29
+ '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
30
+ '.mp3': 'audio/mpeg',
31
+ '.wav': 'audio/wav',
32
+ '.ogg': 'audio/ogg',
33
+ }
34
+
35
+ const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.bmp', '.ico'])
36
+ const VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.mkv'])
37
+ const AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg'])
38
+ const DOCUMENT_EXTS = new Set(['.pdf', '.json', '.csv', '.txt', '.html', '.xml', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx'])
39
+ const ARCHIVE_EXTS = new Set(['.zip', '.tar', '.gz'])
40
+
41
+ export type FileCategory = 'image' | 'video' | 'audio' | 'document' | 'archive' | 'other'
42
+
43
+ export function getFileCategory(ext: string): FileCategory {
44
+ const lower = ext.toLowerCase()
45
+ if (IMAGE_EXTS.has(lower)) return 'image'
46
+ if (VIDEO_EXTS.has(lower)) return 'video'
47
+ if (AUDIO_EXTS.has(lower)) return 'audio'
48
+ if (DOCUMENT_EXTS.has(lower)) return 'document'
49
+ if (ARCHIVE_EXTS.has(lower)) return 'archive'
50
+ return 'other'
51
+ }
@@ -160,8 +160,8 @@ export class OpenClawGateway {
160
160
  this.doConnect().catch(() => {})
161
161
  }, this.reconnectDelay)
162
162
  this.reconnectDelay = Math.min(this.reconnectDelay * 2, maxDelay)
163
- if (this.consecutiveFailures % 5 === 0) {
164
- console.log(`[openclaw-gateway] ${this.consecutiveFailures} consecutive failures, next retry in ${Math.round(this.reconnectDelay / 1000)}s`)
163
+ if (this.consecutiveFailures === 1 || this.consecutiveFailures % 5 === 0) {
164
+ console.log(`[openclaw-gateway] ${this.consecutiveFailures} consecutive failure${this.consecutiveFailures > 1 ? 's' : ''}, next retry in ${Math.round(this.reconnectDelay / 1000)}s`)
165
165
  }
166
166
  }
167
167
 
@@ -11,6 +11,7 @@ import { buildChatModel } from './build-llm'
11
11
  import { getCheckpointSaver } from './langgraph-checkpoint'
12
12
  import { notify } from './ws-hub'
13
13
  import { pushMainLoopEventToMainSessions } from './main-agent-loop'
14
+ import { buildCurrentDateTimePromptContext } from './prompt-runtime-context'
14
15
  import { genId } from '@/lib/id'
15
16
  import { NON_LANGGRAPH_PROVIDER_IDS } from '@/lib/provider-sets'
16
17
  import type { Agent, TaskComment, MessageToolEvent } from '@/types'
@@ -351,6 +352,7 @@ export async function executeLangGraphOrchestrator(
351
352
  const settings = loadSettings()
352
353
  const promptParts: string[] = []
353
354
  if (settings.userPrompt) promptParts.push(settings.userPrompt)
355
+ promptParts.push(buildCurrentDateTimePromptContext())
354
356
  if (orchestrator.soul) promptParts.push(orchestrator.soul)
355
357
  if (orchestrator.systemPrompt) promptParts.push(orchestrator.systemPrompt)
356
358
  // Inject dynamic skills
@@ -6,6 +6,7 @@ import {
6
6
  import { WORKSPACE_DIR } from './data-dir'
7
7
  import { loadRuntimeSettings, getLegacyOrchestratorMaxTurns } from './runtime-settings'
8
8
  import { getMemoryDb } from './memory-db'
9
+ import { buildCurrentDateTimePromptContext } from './prompt-runtime-context'
9
10
  import { getProvider } from '../providers'
10
11
  import type { Agent } from '@/types'
11
12
 
@@ -109,6 +110,7 @@ async function executeOrchestratorLegacy(
109
110
  const settings = loadSettings()
110
111
  const promptParts: string[] = []
111
112
  if (settings.userPrompt) promptParts.push(settings.userPrompt)
113
+ promptParts.push(buildCurrentDateTimePromptContext())
112
114
  if (orchestrator.soul) promptParts.push(orchestrator.soul)
113
115
  if (orchestrator.systemPrompt) promptParts.push(orchestrator.systemPrompt)
114
116
  if (orchestrator.skillIds?.length) {
@@ -308,8 +310,8 @@ async function executeSubTask(
308
310
 
309
311
  export async function callProvider(
310
312
  agent: Agent,
311
- systemPrompt: string,
312
- history: { role: string; text: string }[],
313
+ systemPrompt?: string,
314
+ history: { role: string; text: string }[] = [],
313
315
  ): Promise<string> {
314
316
  const provider = getProvider(agent.provider)
315
317
  if (!provider) throw new Error(`Unknown provider: ${agent.provider}`)
@@ -346,6 +348,7 @@ export async function callProvider(
346
348
  session: mockSession,
347
349
  message: history[history.length - 1].text,
348
350
  apiKey,
351
+ systemPrompt,
349
352
  write: (data: string) => {
350
353
  // Parse SSE data to extract text
351
354
  if (data.startsWith('data: ')) {
@@ -7,9 +7,8 @@
7
7
  import { spawn } from 'child_process'
8
8
  import fs from 'fs'
9
9
  import path from 'path'
10
- import os from 'os'
11
10
 
12
- const UPLOAD_DIR = process.env.SWARMCLAW_UPLOAD_DIR || path.join(os.tmpdir(), 'swarmclaw-uploads')
11
+ const UPLOAD_DIR = process.env.SWARMCLAW_UPLOAD_DIR || path.join(process.env.DATA_DIR || path.join(process.cwd(), 'data'), 'uploads')
13
12
  if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true })
14
13
 
15
14
  const child = spawn('npx', ['@playwright/mcp@latest'], {
@@ -47,7 +46,7 @@ child.stdout.on('data', (chunk) => {
47
46
  fs.writeFileSync(path.join(UPLOAD_DIR, filename), Buffer.from(block.data, 'base64'))
48
47
  newContent.push({
49
48
  type: 'text',
50
- text: `Screenshot saved. Show it to the user with this markdown: ![Screenshot](/api/uploads/${filename})`,
49
+ text: `Screenshot saved to /api/uploads/${filename} — it is already displayed inline above (do not repeat it with markdown).`,
51
50
  })
52
51
  newContent.push(block) // keep image so Claude can see it
53
52
  } else {
@@ -0,0 +1,53 @@
1
+ function resolveLocalTimezone(): string {
2
+ try {
3
+ return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC'
4
+ } catch {
5
+ return 'UTC'
6
+ }
7
+ }
8
+
9
+ function formatDateTimeInTimezone(date: Date, timezone: string): string | null {
10
+ try {
11
+ return new Intl.DateTimeFormat('en-US', {
12
+ weekday: 'long',
13
+ year: 'numeric',
14
+ month: 'long',
15
+ day: 'numeric',
16
+ hour: '2-digit',
17
+ minute: '2-digit',
18
+ second: '2-digit',
19
+ hour12: false,
20
+ timeZone: timezone,
21
+ timeZoneName: 'short',
22
+ }).format(date)
23
+ } catch {
24
+ return null
25
+ }
26
+ }
27
+
28
+ export function buildCurrentDateTimePromptContext(preferredTimezone?: string | null): string {
29
+ const now = new Date()
30
+ const utcIso = now.toISOString()
31
+ const utcFormatted = formatDateTimeInTimezone(now, 'UTC') || utcIso
32
+ const localTimezone = resolveLocalTimezone()
33
+ const requestedTimezone = (preferredTimezone || '').trim()
34
+ const chosenTimezone = requestedTimezone || localTimezone
35
+ const chosenFormatted = formatDateTimeInTimezone(now, chosenTimezone)
36
+
37
+ const lines = [
38
+ '## Runtime Date/Time Context',
39
+ `- Current timestamp (UTC): ${utcIso}`,
40
+ `- Current date/time (UTC): ${utcFormatted}`,
41
+ ]
42
+
43
+ if (chosenFormatted) {
44
+ lines.push(`- Current date/time (${chosenTimezone}): ${chosenFormatted}`)
45
+ } else if (requestedTimezone) {
46
+ lines.push(`- Requested timezone "${requestedTimezone}" could not be resolved. Use UTC time above.`)
47
+ }
48
+
49
+ lines.push('- Treat these as authoritative for terms like "today", "yesterday", "tomorrow", and "recent".')
50
+ lines.push('- For time-sensitive answers, use explicit dates (for example, "March 2, 2026").')
51
+
52
+ return lines.join('\n')
53
+ }
@@ -13,13 +13,13 @@ import { isProtectedMainSession } from './main-session'
13
13
  import type { Agent, BoardTask, Message } from '@/types'
14
14
 
15
15
  // HMR-safe: pin processing flag to globalThis so hot reloads don't reset it
16
- const _queueState = ((globalThis as Record<string, unknown>).__swarmclaw_queue__ ??= { processing: false }) as { processing: boolean }
16
+ const _queueState = ((globalThis as Record<string, unknown>).__swarmclaw_queue__ ??= { processing: false, pendingKick: false }) as { processing: boolean; pendingKick: boolean }
17
17
 
18
18
  interface SessionMessageLike {
19
19
  role?: string
20
20
  text?: string
21
21
  time?: number
22
- kind?: 'chat' | 'heartbeat' | 'system'
22
+ kind?: 'chat' | 'heartbeat' | 'system' | 'context-clear'
23
23
  toolEvents?: Array<{ name?: string; output?: string }>
24
24
  }
25
25
 
@@ -280,19 +280,44 @@ function notifyAgentThreadTaskResult(task: BoardTask): void {
280
280
  changed = true
281
281
  }
282
282
 
283
- // 2. If delegated, push to delegating agent's thread
283
+ // 2. If delegated, push to delegating agent's thread AND active chat sessions
284
284
  const delegatedBy = (task as unknown as Record<string, unknown>).delegatedByAgentId
285
285
  if (typeof delegatedBy === 'string' && delegatedBy !== task.agentId) {
286
286
  const delegator = agents[delegatedBy]
287
+ const agentName = agent?.name || task.agentId
288
+ const delegationBody = buildResultBlock(`Delegated task ${statusLabel}: **${taskLink}** (by ${agentName})`)
289
+
290
+ // Push to delegating agent's thread
287
291
  if (delegator?.threadSessionId && sessions[delegator.threadSessionId]) {
288
292
  const thread = sessions[delegator.threadSessionId]
289
293
  if (!Array.isArray(thread.messages)) thread.messages = []
290
- const agentName = agent?.name || task.agentId
291
- const body = buildResultBlock(`Delegated task ${statusLabel}: **${taskLink}** (by ${agentName})`)
292
- thread.messages.push(buildMsg(body))
294
+ thread.messages.push(buildMsg(delegationBody))
293
295
  thread.lastActiveAt = now
294
296
  changed = true
295
297
  }
298
+
299
+ // Push to delegating agent's active user-facing chat sessions
300
+ // so the result is visible in the chat the user is looking at
301
+ if (delegator) {
302
+ for (const session of Object.values(sessions)) {
303
+ if (!session || session.agentId !== delegatedBy) continue
304
+ // Skip thread sessions and orchestrated/subagent sessions
305
+ if (session.id === delegator.threadSessionId) continue
306
+ if (session.sessionType === 'orchestrated') continue
307
+ // Only push to recently-active sessions (within last 30 minutes)
308
+ const lastActive = typeof session.lastActiveAt === 'number' ? session.lastActiveAt : 0
309
+ if (now - lastActive > 30 * 60_000) continue
310
+ if (!Array.isArray(session.messages)) session.messages = []
311
+ // Avoid duplicate push
312
+ const lastMsg = session.messages.at(-1)
313
+ if (lastMsg?.text === delegationBody && typeof lastMsg?.time === 'number' && now - lastMsg.time < 30_000) continue
314
+ session.messages.push(buildMsg(delegationBody))
315
+ session.lastActiveAt = now
316
+ changed = true
317
+ // Notify the specific session's message topic for real-time UI update
318
+ notify(`messages:${session.id}`)
319
+ }
320
+ }
296
321
  }
297
322
 
298
323
  if (changed) saveSessions(sessions)
@@ -331,6 +356,10 @@ export function enqueueTask(taskId: string) {
331
356
  text: `Task queued: "${task.title}" (${task.id})`,
332
357
  })
333
358
 
359
+ // If processNext is already running, mark a pending kick so it re-enters after finishing
360
+ if (_queueState.processing) {
361
+ _queueState.pendingKick = true
362
+ }
334
363
  // Delay before kicking worker so UI shows the queued state
335
364
  setTimeout(() => processNext(), 2000)
336
365
  }
@@ -619,10 +648,21 @@ export async function processNext() {
619
648
  // Save initial assistant message so user sees context when opening the session
620
649
  const sessions = loadSessions()
621
650
  if (sessions[sessionId]) {
651
+ const isDelegation = (task as unknown as Record<string, unknown>).sourceType === 'delegation'
652
+ let initialText: string
653
+ if (isDelegation) {
654
+ const delegatorId = (task as unknown as Record<string, unknown>).delegatedByAgentId as string | undefined
655
+ const delegator = delegatorId ? agents[delegatorId] : null
656
+ const prefix = `[delegation-source:${delegatorId || ''}:${delegator?.name || 'Agent'}:${delegator?.avatarSeed || ''}]`
657
+ initialText = `${prefix}\nDelegated by **${delegator?.name || 'another agent'}** | [${task.title}](#task:${task.id})\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`\n\nI'll begin working on this now.`
658
+ } else {
659
+ initialText = `Starting task: **${task.title}**\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`\n\nI'll begin working on this now.`
660
+ }
622
661
  sessions[sessionId].messages.push({
623
662
  role: 'assistant',
624
- text: `Starting task: **${task.title}**\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`\n\nI'll begin working on this now.`,
663
+ text: initialText,
625
664
  time: Date.now(),
665
+ ...(isDelegation ? { kind: 'system' as const } : {}),
626
666
  })
627
667
  saveSessions(sessions)
628
668
  }
@@ -807,6 +847,11 @@ export async function processNext() {
807
847
  }
808
848
  } finally {
809
849
  _queueState.processing = false
850
+ // If tasks were enqueued while we were processing, kick another round
851
+ if (_queueState.pendingKick) {
852
+ _queueState.pendingKick = false
853
+ setTimeout(() => processNext(), 500)
854
+ }
810
855
  }
811
856
  }
812
857
 
@@ -0,0 +1,67 @@
1
+ import { z } from 'zod'
2
+ import { tool, type StructuredToolInterface } from '@langchain/core/tools'
3
+ import { loadSessions, saveSessions } from '../storage'
4
+ import { notify } from '../ws-hub'
5
+ import type { ToolBuildContext } from './context'
6
+
7
+ export function buildCanvasTools(bctx: ToolBuildContext): StructuredToolInterface[] {
8
+ const { ctx, hasTool } = bctx
9
+ if (!hasTool('canvas')) return []
10
+
11
+ return [
12
+ tool(
13
+ async ({ action, content }) => {
14
+ try {
15
+ const sessionId = ctx?.sessionId
16
+ if (!sessionId) return 'Error: no active session for canvas.'
17
+
18
+ const sessions = loadSessions()
19
+ const session = sessions[sessionId]
20
+ if (!session) return 'Error: session not found.'
21
+
22
+ if (action === 'present') {
23
+ if (!content) return 'Error: content is required for present action.'
24
+ ;(session as Record<string, unknown>).canvasContent = content
25
+ session.lastActiveAt = Date.now()
26
+ sessions[sessionId] = session
27
+ saveSessions(sessions)
28
+ notify(`canvas:${sessionId}`)
29
+ return JSON.stringify({ ok: true, action: 'present', contentLength: content.length })
30
+ }
31
+
32
+ if (action === 'hide') {
33
+ ;(session as Record<string, unknown>).canvasContent = null
34
+ session.lastActiveAt = Date.now()
35
+ sessions[sessionId] = session
36
+ saveSessions(sessions)
37
+ notify(`canvas:${sessionId}`)
38
+ return JSON.stringify({ ok: true, action: 'hide' })
39
+ }
40
+
41
+ if (action === 'snapshot') {
42
+ const current = (session as Record<string, unknown>).canvasContent
43
+ return JSON.stringify({
44
+ ok: true,
45
+ action: 'snapshot',
46
+ hasContent: !!current,
47
+ contentLength: typeof current === 'string' ? current.length : 0,
48
+ preview: typeof current === 'string' ? current.slice(0, 500) : null,
49
+ })
50
+ }
51
+
52
+ return `Unknown canvas action "${action}". Valid: present, hide, snapshot`
53
+ } catch (err: unknown) {
54
+ return `Error: ${err instanceof Error ? err.message : String(err)}`
55
+ }
56
+ },
57
+ {
58
+ name: 'canvas',
59
+ description: 'Present live HTML/CSS/JS content to the user in an interactive canvas panel. Use "present" to show content, "hide" to dismiss, "snapshot" to check current state. The canvas renders in a sandboxed iframe alongside the chat.',
60
+ schema: z.object({
61
+ action: z.enum(['present', 'hide', 'snapshot']).describe('Canvas action to perform'),
62
+ content: z.string().optional().describe('HTML content to render (required for "present"). Can include inline CSS and JS.'),
63
+ }),
64
+ },
65
+ ),
66
+ ]
67
+ }
@@ -1,8 +1,21 @@
1
1
  import { z } from 'zod'
2
2
  import { tool, type StructuredToolInterface } from '@langchain/core/tools'
3
- import { loadConnectors, loadSettings } from '../storage'
3
+ import path from 'path'
4
+ import fs from 'fs'
5
+ import { loadConnectors, loadSettings, UPLOAD_DIR } from '../storage'
4
6
  import type { ToolBuildContext } from './context'
5
7
 
8
+ /** Resolve /api/uploads/filename URLs to actual disk paths */
9
+ function resolveUploadUrl(url: string | undefined): { mediaPath: string; mimeType?: string } | null {
10
+ if (!url) return null
11
+ const match = url.match(/^\/api\/uploads\/([^?#]+)/)
12
+ if (!match) return null
13
+ const safeName = match[1].replace(/[^a-zA-Z0-9._-]/g, '')
14
+ const filePath = path.join(UPLOAD_DIR, safeName)
15
+ if (!fs.existsSync(filePath)) return null
16
+ return { mediaPath: filePath }
17
+ }
18
+
6
19
  export function buildConnectorTools(bctx: ToolBuildContext): StructuredToolInterface[] {
7
20
  const tools: StructuredToolInterface[] = []
8
21
  const { ctx, hasTool } = bctx
@@ -32,6 +45,29 @@ export function buildConnectorTools(bctx: ToolBuildContext): StructuredToolInter
32
45
  return JSON.stringify(running)
33
46
  }
34
47
 
48
+ if (action === 'start') {
49
+ if (!connectorId) {
50
+ // If no ID given, list available connectors to start
51
+ const allConnectors = loadConnectors()
52
+ const stopped = Object.values(allConnectors)
53
+ .filter((c) => !platform || c.platform === platform)
54
+ .filter((c) => !running.find((r) => r.id === c.id))
55
+ .map((c) => ({ id: c.id, name: c.name, platform: c.platform }))
56
+ if (!stopped.length) return 'All connectors are already running.'
57
+ return `Error: connectorId is required. Stopped connectors available to start: ${JSON.stringify(stopped)}`
58
+ }
59
+ const { startConnector: doStart } = await import('../connectors/manager')
60
+ await doStart(connectorId)
61
+ return JSON.stringify({ status: 'started', connectorId })
62
+ }
63
+
64
+ if (action === 'stop') {
65
+ if (!connectorId) return 'Error: connectorId is required for stop action.'
66
+ const { stopConnector: doStop } = await import('../connectors/manager')
67
+ await doStop(connectorId)
68
+ return JSON.stringify({ status: 'stopped', connectorId })
69
+ }
70
+
35
71
  if (action === 'send') {
36
72
  const settings = loadSettings()
37
73
  if (settings.safetyRequireApprovalForOutbound === true && approved !== true) {
@@ -41,7 +77,15 @@ export function buildConnectorTools(bctx: ToolBuildContext): StructuredToolInter
41
77
  const hasMedia = !!imageUrl?.trim() || !!fileUrl?.trim()
42
78
  if (!hasText && !hasMedia) return 'Error: message or media URL is required for send action.'
43
79
  if (!running.length) {
44
- return `Error: no running connectors${platform ? ` for platform "${platform}"` : ''}.`
80
+ // Check for configured-but-not-running connectors to give actionable feedback
81
+ const allConnectors = loadConnectors()
82
+ const configured = Object.values(allConnectors)
83
+ .filter((c) => !platform || c.platform === platform)
84
+ .map((c) => ({ id: c.id, name: c.name, platform: c.platform, agentId: c.agentId || null }))
85
+ if (configured.length) {
86
+ return `Error: no running connectors${platform ? ` for platform "${platform}"` : ''}, but ${configured.length} configured connector(s) found: ${JSON.stringify(configured)}. These connectors exist but are not currently started. Ask the user if they'd like you to start one (use action "start" with the connectorId), then retry the send.`
87
+ }
88
+ return `Error: no running connectors${platform ? ` for platform "${platform}"` : ''}. No connectors are configured for this platform either — the user needs to set one up in the Connectors panel first.`
45
89
  }
46
90
 
47
91
  const selected = connectorId
@@ -75,19 +119,49 @@ export function buildConnectorTools(bctx: ToolBuildContext): StructuredToolInter
75
119
  if (allowed.length) channelId = allowed[0]
76
120
  }
77
121
  if (!channelId) {
78
- return `Error: no target recipient configured. Provide "to", or set connector config "outboundJid"/"allowedJids"/"outboundTarget"/"allowFrom".`
122
+ // Collect any known numbers/targets from config to help the agent suggest them
123
+ const knownTargets: string[] = []
124
+ const jids = connector.config?.allowedJids?.split(',').map((s: string) => s.trim()).filter(Boolean) || []
125
+ const from = connector.config?.allowFrom?.split(',').map((s: string) => s.trim()).filter(Boolean) || []
126
+ const outJid = connector.config?.outboundJid?.trim()
127
+ const outTarget = connector.config?.outboundTarget?.trim()
128
+ if (outJid) knownTargets.push(outJid)
129
+ if (outTarget) knownTargets.push(outTarget)
130
+ knownTargets.push(...jids, ...from)
131
+ const unique = [...new Set(knownTargets)]
132
+ if (unique.length) {
133
+ return `Error: no default outbound target is set, but the connector has ${unique.length} configured number(s)/target(s): ${JSON.stringify(unique)}. Ask the user which one to send to, then re-call with the "to" parameter set to their choice.`
134
+ }
135
+ return `Error: no target recipient configured and no known contacts on this connector. Ask the user for the recipient number/ID, then re-call with the "to" parameter. They can also configure "allowedJids" or "outboundJid" in the connector settings.`
79
136
  }
80
137
  if (connector.platform === 'whatsapp') {
81
138
  channelId = normalizeWhatsAppTarget(channelId)
82
139
  }
83
140
 
141
+ // Resolve /api/uploads/ URLs to actual disk paths so connectors can read the files
142
+ let resolvedMediaPath = mediaPath?.trim() || undefined
143
+ let resolvedImageUrl = imageUrl?.trim() || undefined
144
+ let resolvedFileUrl = fileUrl?.trim() || undefined
145
+ if (!resolvedMediaPath) {
146
+ const fromImage = resolveUploadUrl(resolvedImageUrl)
147
+ if (fromImage) {
148
+ resolvedMediaPath = fromImage.mediaPath
149
+ resolvedImageUrl = undefined
150
+ }
151
+ const fromFile = resolveUploadUrl(resolvedFileUrl)
152
+ if (fromFile) {
153
+ resolvedMediaPath = fromFile.mediaPath
154
+ resolvedFileUrl = undefined
155
+ }
156
+ }
157
+
84
158
  const sent = await sendConnectorMessage({
85
159
  connectorId: selected.id,
86
160
  channelId,
87
161
  text: message?.trim() || '',
88
- imageUrl: imageUrl?.trim() || undefined,
89
- fileUrl: fileUrl?.trim() || undefined,
90
- mediaPath: mediaPath?.trim() || undefined,
162
+ imageUrl: resolvedImageUrl,
163
+ fileUrl: resolvedFileUrl,
164
+ mediaPath: resolvedMediaPath,
91
165
  mimeType: mimeType?.trim() || undefined,
92
166
  fileName: fileName?.trim() || undefined,
93
167
  caption: caption?.trim() || undefined,
@@ -140,16 +214,16 @@ export function buildConnectorTools(bctx: ToolBuildContext): StructuredToolInter
140
214
  }
141
215
  }
142
216
 
143
- return 'Unknown action. Use list_running, list_targets, or send.'
217
+ return 'Unknown action. Use list_running, list_targets, start, stop, or send.'
144
218
  } catch (err: unknown) {
145
219
  return `Error: ${err instanceof Error ? err.message : String(err)}`
146
220
  }
147
221
  },
148
222
  {
149
223
  name: 'connector_message_tool',
150
- description: 'Send proactive outbound messages and perform rich messaging actions through running connectors. Supports listing running connectors/targets, sending text/media, and rich messaging (react, edit, delete, pin). For rich actions: connectorId + message (as messageId) required; caption carries emoji for react or new text for edit.',
224
+ description: 'Manage and send messages through chat platform connectors (WhatsApp, Telegram, Slack, Discord, etc.). Use "start"/"stop" to manage connector lifecycle, "list_running"/"list_targets" to discover available connectors and recipients, "send" to deliver messages, and rich actions (react, edit, delete, pin) for message management. When a send fails because no connector is running, check if one is configured and offer to start it. When no target is set, list available configured numbers and ask the user which to send to.',
151
225
  schema: z.object({
152
- action: z.enum(['list_running', 'list_targets', 'send', 'message_react', 'message_edit', 'message_delete', 'message_pin']).describe('connector messaging action'),
226
+ action: z.enum(['list_running', 'list_targets', 'start', 'stop', 'send', 'message_react', 'message_edit', 'message_delete', 'message_pin']).describe('connector messaging action'),
153
227
  connectorId: z.string().optional().describe('Optional connector id. Defaults to the first running connector (or first for selected platform).'),
154
228
  platform: z.string().optional().describe('Optional platform filter (whatsapp, telegram, slack, discord, bluebubbles, etc.).'),
155
229
  to: z.string().optional().describe('Target channel id / recipient. For WhatsApp, phone number or full JID.'),
@@ -20,6 +20,8 @@ import {
20
20
  } from '../storage'
21
21
  import { resolveScheduleName } from '@/lib/schedule-name'
22
22
  import { findDuplicateSchedule, type ScheduleLike } from '@/lib/schedule-dedupe'
23
+ import { computeTaskFingerprint, findDuplicateTask } from '@/lib/task-dedupe'
24
+ import { resolveTaskAgentFromDescription } from '@/lib/server/task-mention'
23
25
  import type { ToolBuildContext } from './context'
24
26
  import { safePath, findBinaryOnPath } from './context'
25
27
 
@@ -115,6 +117,7 @@ const RESOURCE_DEFAULTS: Record<string, (parsed: any) => any> = {
115
117
  queuedAt: null,
116
118
  startedAt: null,
117
119
  completedAt: null,
120
+ priority: ['low', 'medium', 'high', 'critical'].includes(p.priority) ? p.priority : undefined,
118
121
  ...p,
119
122
  }),
120
123
  manage_schedules: (p) => {
@@ -342,6 +345,24 @@ export function buildCrudTools(bctx: ToolBuildContext): StructuredToolInterface[
342
345
  })
343
346
  }
344
347
  }
348
+ // @mention agent resolution for tasks
349
+ if (toolKey === 'manage_tasks' && parsed.description) {
350
+ const agents = loadAgents()
351
+ parsed.agentId = resolveTaskAgentFromDescription(
352
+ parsed.description,
353
+ parsed.agentId || ctx?.agentId || '',
354
+ agents,
355
+ )
356
+ }
357
+ // Task dedup
358
+ if (toolKey === 'manage_tasks') {
359
+ const fp = computeTaskFingerprint(parsed.title || 'Untitled Task', parsed.agentId || ctx?.agentId || '')
360
+ parsed.fingerprint = fp
361
+ const dupe = findDuplicateTask(all as Record<string, import('@/types').BoardTask>, { fingerprint: fp })
362
+ if (dupe) {
363
+ return JSON.stringify({ ...dupe, deduplicated: true })
364
+ }
365
+ }
345
366
  const newId = genId()
346
367
  const entry = {
347
368
  id: newId,
@@ -634,6 +634,69 @@ export function buildDelegateTools(bctx: ToolBuildContext): StructuredToolInterf
634
634
  }
635
635
  }
636
636
 
637
+ // check_delegation_status: lets agents check on tasks they delegated
638
+ if (ctx?.platformAssignScope === 'all' && ctx?.agentId) {
639
+ tools.push(
640
+ tool(
641
+ async ({ taskId }) => {
642
+ try {
643
+ const tasks = loadTasks()
644
+ const task = tasks[taskId] as Record<string, unknown> | undefined
645
+ if (!task) return `Error: Task "${taskId}" not found.`
646
+
647
+ const status = task.status as string || 'unknown'
648
+ const result = typeof task.result === 'string' ? task.result : null
649
+ const error = typeof task.error === 'string' ? task.error : null
650
+ const agentId = task.agentId as string || ''
651
+ const agents = loadAgents()
652
+ const agent = agents[agentId]
653
+ const startedAt = typeof task.startedAt === 'number' ? task.startedAt : null
654
+ const completedAt = typeof task.completedAt === 'number' ? task.completedAt : null
655
+
656
+ const info: Record<string, unknown> = {
657
+ taskId,
658
+ status,
659
+ agentId,
660
+ agentName: agent?.name || agentId,
661
+ agentAvatarSeed: agent?.avatarSeed || null,
662
+ title: task.title || '',
663
+ }
664
+
665
+ if (startedAt) info.startedAt = new Date(startedAt).toISOString()
666
+ if (completedAt) info.completedAt = new Date(completedAt).toISOString()
667
+ if (startedAt && !completedAt && status === 'running') {
668
+ info.runningForSeconds = Math.round((Date.now() - startedAt) / 1000)
669
+ }
670
+ if (result) info.result = result.slice(0, 4000)
671
+ if (error) info.error = error.slice(0, 1000)
672
+
673
+ // Include latest comments for context
674
+ const comments = Array.isArray(task.comments) ? task.comments as Array<{ text: string; author: string; createdAt: number }> : []
675
+ if (comments.length > 0) {
676
+ const latest = comments.slice(-3).map((c) => ({
677
+ author: c.author,
678
+ text: (c.text || '').slice(0, 500),
679
+ time: new Date(c.createdAt).toISOString(),
680
+ }))
681
+ info.latestComments = latest
682
+ }
683
+
684
+ return JSON.stringify(info)
685
+ } catch (err: unknown) {
686
+ return `Error checking task: ${err instanceof Error ? err.message : String(err)}`
687
+ }
688
+ },
689
+ {
690
+ name: 'check_delegation_status',
691
+ description: 'Check the status and result of a delegated task. Use this after delegate_to_agent to monitor progress. Returns status (todo/queued/running/completed/failed), result if completed, and latest comments.',
692
+ schema: z.object({
693
+ taskId: z.string().describe('The task ID returned by delegate_to_agent'),
694
+ }),
695
+ },
696
+ ),
697
+ )
698
+ }
699
+
637
700
  // delegate_to_agent: requires "Assign to Other Agents" (platformAssignScope: 'all')
638
701
  if (ctx?.platformAssignScope === 'all' && ctx?.agentId) {
639
702
  tools.push(
@@ -698,9 +761,10 @@ export function buildDelegateTools(bctx: ToolBuildContext): StructuredToolInterf
698
761
  taskId,
699
762
  agentId: resolvedId,
700
763
  agentName: target.name,
764
+ agentAvatarSeed: target.avatarSeed || null,
701
765
  message: startImmediately
702
- ? `Task delegated to ${target.name} and queued for immediate execution. Task ID: ${taskId}.`
703
- : `Task delegated to ${target.name}. Task ID: ${taskId}. Status: todo. Ask the user if they want to start it now — call again with startImmediately: true to queue it.`,
766
+ ? `Task delegated to ${target.name} and queued for immediate execution. Task ID: ${taskId}. Use check_delegation_status to monitor progress.`
767
+ : `Task delegated to ${target.name}. Task ID: ${taskId}. Status: todo (not auto-started). Use delegate_to_agent with startImmediately: true to queue it.`,
704
768
  })
705
769
  } catch (err: unknown) {
706
770
  return `Error delegating task: ${err instanceof Error ? err.message : String(err)}`
@@ -708,12 +772,12 @@ export function buildDelegateTools(bctx: ToolBuildContext): StructuredToolInterf
708
772
  },
709
773
  {
710
774
  name: 'delegate_to_agent',
711
- description: 'Delegate a task to another agent. Creates a task on the task board. By default the task goes to "todo" status. Set startImmediately=true to queue it for execution right away. Ask the user to confirm before starting immediately.',
775
+ description: 'Delegate a task to another agent. Creates a task on the task board and queues it for immediate execution by default. Set startImmediately=false if you want the task to go to "todo" status instead.',
712
776
  schema: z.object({
713
777
  agentId: z.string().describe('ID or name of the target agent to delegate to'),
714
778
  task: z.string().describe('What the target agent should do'),
715
779
  description: z.string().optional().describe('Optional longer description of the task'),
716
- startImmediately: z.boolean().optional().default(false).describe('If true, queue the task for immediate execution instead of putting it in todo'),
780
+ startImmediately: z.boolean().optional().default(true).describe('If true (default), queue the task for immediate execution. Set false to put in todo for manual start.'),
717
781
  }),
718
782
  },
719
783
  ),