@swarmclawai/swarmclaw 0.5.3 → 0.6.2

package/src/lib/providers/openai.ts

@@ -43,9 +43,9 @@ async function fileToContentParts(filePath: string): Promise<any[]> {
   return [{ type: 'text', text: `[Attached file: ${name}]` }]
 }
 
-export function streamOpenAiChat({ session, message, imagePath, apiKey, systemPrompt, write, active, loadHistory, onUsage }: StreamChatOptions): Promise<string> {
+export function streamOpenAiChat({ session, message, imagePath, imageUrl, apiKey, systemPrompt, write, active, loadHistory, onUsage }: StreamChatOptions): Promise<string> {
   return new Promise(async (resolve) => {
-    const messages = await buildMessages(session, message, imagePath, systemPrompt, loadHistory)
+    const messages = await buildMessages(session, message, imagePath, systemPrompt, loadHistory, imageUrl)
     const model = session.model || 'gpt-4o'
 
     const payload = JSON.stringify({
@@ -163,7 +163,11 @@ export function streamOpenAiChat({ session, message, imagePath, apiKey, systemPr
   })
 }
 
-async function buildMessages(session: any, message: string, imagePath: string | undefined, systemPrompt: string | undefined, loadHistory: (id: string) => any[]) {
+function urlToImagePart(url: string): { type: string; image_url: { url: string; detail: string } } {
+  return { type: 'image_url', image_url: { url, detail: 'auto' } }
+}
+
+async function buildMessages(session: any, message: string, imagePath: string | undefined, systemPrompt: string | undefined, loadHistory: (id: string) => any[], imageUrl?: string) {
   const msgs: Array<{ role: string; content: any }> = []
 
   if (systemPrompt) {
@@ -173,8 +177,9 @@ async function buildMessages(session: any, message: string, imagePath: string |
   if (loadHistory) {
     const history = loadHistory(session.id).slice(-40)
     for (const m of history) {
-      if (m.role === 'user' && m.imagePath) {
-        const parts = await fileToContentParts(m.imagePath)
+      if (m.role === 'user' && (m.imagePath || m.imageUrl)) {
+        const parts = m.imagePath ? await fileToContentParts(m.imagePath) : []
+        if (m.imageUrl) parts.push(urlToImagePart(m.imageUrl))
         msgs.push({ role: 'user', content: [...parts, { type: 'text', text: m.text }] })
       } else {
         msgs.push({ role: m.role, content: m.text })
@@ -183,8 +188,9 @@ async function buildMessages(session: any, message: string, imagePath: string |
   }
 
   // Current message with optional attachment
-  if (imagePath) {
-    const parts = await fileToContentParts(imagePath)
+  if (imagePath || imageUrl) {
+    const parts = imagePath ? await fileToContentParts(imagePath) : []
+    if (imageUrl) parts.push(urlToImagePart(imageUrl))
     msgs.push({ role: 'user', content: [...parts, { type: 'text', text: message }] })
   } else {
     msgs.push({ role: 'user', content: message })

package/src/lib/server/chat-execution.ts

@@ -24,11 +24,20 @@ import { getMemoryDb } from './memory-db'
 import { routeTaskIntent } from './capability-router'
 import { notify } from './ws-hub'
 import { resolveConcreteToolPolicyBlock, resolveSessionToolPolicy } from './tool-capability-policy'
-import type { MessageToolEvent, SSEEvent, UsageRecord } from '@/types'
+import { buildCurrentDateTimePromptContext } from './prompt-runtime-context'
+import type { Message, MessageToolEvent, SSEEvent, UsageRecord } from '@/types'
 import { markProviderFailure, markProviderSuccess, rankDelegatesByHealth } from './provider-health'
 import { NON_LANGGRAPH_PROVIDER_IDS } from '@/lib/provider-sets'
 type DelegateTool = 'delegate_to_claude_code' | 'delegate_to_codex_cli' | 'delegate_to_opencode_cli'
 
+/** Slice history from the most recent context-clear marker forward */
+function applyContextClearBoundary(messages: Message[]): Message[] {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i].kind === 'context-clear') return messages.slice(i + 1)
+  }
+  return messages
+}
+
 interface SessionWithTools {
   tools?: string[] | null
 }
@@ -55,6 +64,7 @@ export interface ExecuteChatTurnInput {
   onEvent?: (event: SSEEvent) => void
   modelOverride?: string
   heartbeatConfig?: { ackMaxChars: number; showOk: boolean; showAlerts: boolean; target: string | null }
+  replyToId?: string
 }
 
 export interface ExecuteChatTurnResult {
@@ -306,11 +316,11 @@ function buildAgentSystemPrompt(session: any): string | undefined {
   if (!session.agentId) return undefined
   const agents = loadAgents()
   const agent = agents[session.agentId]
-  if (!agent?.systemPrompt && !agent?.soul) return undefined
 
   const settings = loadSettings()
   const parts: string[] = []
   if (settings.userPrompt) parts.push(settings.userPrompt)
+  parts.push(buildCurrentDateTimePromptContext())
   if (agent.soul) parts.push(agent.soul)
   if (agent.systemPrompt) parts.push(agent.systemPrompt)
   if (agent.skillIds?.length) {
@@ -320,6 +330,7 @@ function buildAgentSystemPrompt(session: any): string | undefined {
       if (skill?.content) parts.push(`## Skill: ${skill.name}\n${skill.content}`)
     }
   }
+  if (!parts.length) return undefined
   return parts.join('\n\n')
 }
 
@@ -341,13 +352,26 @@ function resolveApiKeyForSession(session: SessionWithCredentials, provider: Prov
   return null
 }
 
-function classifyHeartbeatResponse(text: string, ackMaxChars: number): 'suppress' | 'strip' | 'keep' {
-  const trimmed = text.trim()
-  if (trimmed === 'HEARTBEAT_OK' || trimmed === 'NO_MESSAGE') return 'suppress'
-  const stripped = trimmed.replace(/HEARTBEAT_OK/gi, '').replace(/NO_MESSAGE/gi, '').trim()
+function stripMarkupForHeartbeat(text: string): string {
+  return text
+    .replace(/<[^>]*>/g, ' ')       // strip HTML tags
+    .replace(/&nbsp;/gi, ' ')       // decode nbsp
+    .replace(/^[*`~_]+/, '')        // strip leading markdown
+    .replace(/[*`~_]+$/, '')        // strip trailing markdown
+    .trim()
+}
+
+const HEARTBEAT_OK_RE = /HEARTBEAT_OK[^\w]{0,4}$/
+const NO_MESSAGE_RE = /NO_MESSAGE[^\w]{0,4}$/
+
+function classifyHeartbeatResponse(text: string, ackMaxChars: number, hadToolCalls: boolean): 'suppress' | 'strip' | 'keep' {
+  const cleaned = stripMarkupForHeartbeat(text)
+  if (cleaned === 'HEARTBEAT_OK' || cleaned === 'NO_MESSAGE') return 'suppress'
+  if (HEARTBEAT_OK_RE.test(cleaned) || NO_MESSAGE_RE.test(cleaned)) return 'suppress'
+  const stripped = cleaned.replace(/HEARTBEAT_OK/gi, '').replace(/NO_MESSAGE/gi, '').trim()
   if (!stripped) return 'suppress'
-  if (stripped.length <= ackMaxChars) return 'suppress'
-  return stripped.length < trimmed.length ? 'strip' : 'keep'
+  if (!hadToolCalls && stripped.length <= ackMaxChars) return 'suppress'
+  return stripped.length < cleaned.length ? 'strip' : 'keep'
 }
 
 function estimateConversationTone(text: string): string {
@@ -542,6 +566,7 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
       imagePath: imagePath || undefined,
       imageUrl: imageUrl || undefined,
       attachedFiles: attachedFiles?.length ? attachedFiles : undefined,
+      replyToId: input.replyToId || undefined,
     })
     session.lastActiveAt = Date.now()
     saveSessions(sessions)
@@ -551,6 +576,7 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
   const toolEvents: MessageToolEvent[] = []
   const streamErrors: string[] = []
 
+  let thinkingText = ''
   const emit = (ev: SSEEvent) => {
     if (ev.t === 'err' && typeof ev.text === 'string') {
       const trimmed = ev.text.trim()
@@ -559,6 +585,9 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
         if (streamErrors.length > 8) streamErrors.shift()
       }
     }
+    if (ev.t === 'thinking' && ev.text) {
+      thinkingText += ev.text
+    }
     collectToolEvent(ev, toolEvents)
     onEvent?.(ev)
   }
@@ -593,9 +622,12 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
   const hasTools = !!sessionForRun.tools?.length && !NON_LANGGRAPH_PROVIDER_IDS.has(providerType)
 
   try {
-    // Heartbeat runs are self-contained — skip conversation history to avoid
-    // blowing past the context window on long-lived sessions.
-    const heartbeatHistory = isAutoRunNoHistory ? [] : undefined
+    // Heartbeat runs get a small tail of recent messages so the agent can see
+    // prior findings and avoid repeating the same searches. Full history is
+    // skipped to avoid blowing the context window on long-lived sessions.
+    const heartbeatHistory = isAutoRunNoHistory
+      ? getSessionMessages(sessionId).slice(-6)
+      : undefined
 
     console.log(`[chat-execution] provider=${providerType}, hasTools=${hasTools}, imagePath=${imagePath || 'none'}, attachedFiles=${attachedFiles?.length || 0}, tools=${(sessionForRun.tools || []).length}`)
 
@@ -608,7 +640,7 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
           apiKey,
           systemPrompt,
           write: (raw) => parseAndEmit(raw),
-          history: heartbeatHistory ?? getSessionMessages(sessionId),
+          history: heartbeatHistory ?? applyContextClearBoundary(getSessionMessages(sessionId)),
           signal: abortController.signal,
         })).fullText
       : await provider.handler.streamChat({
@@ -619,7 +651,7 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
           systemPrompt,
           write: (raw: string) => parseAndEmit(raw),
           active,
-          loadHistory: isAutoRunNoHistory ? () => [] : getSessionMessages,
+          loadHistory: isAutoRunNoHistory ? () => getSessionMessages(sessionId).slice(-6) : (sid: string) => applyContextClearBoundary(getSessionMessages(sid)),
           onUsage: (u) => { directUsage.inputTokens = u.inputTokens; directUsage.outputTokens = u.outputTokens; directUsage.received = true },
         })
   } catch (err: any) {
@@ -708,7 +740,13 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
       const toolOutput = await selectedTool.invoke(args)
       const outputText = typeof toolOutput === 'string' ? toolOutput : JSON.stringify(toolOutput)
       emit({ t: 'tool_result', toolName, toolOutput: outputText })
-      if (outputText?.trim()) fullResponse = outputText.trim()
+      // Don't overwrite fullResponse with raw tool output — it's already captured
+      // in toolEvents. Only set a brief notice when the LLM produced no text,
+      // so the message bubble isn't empty.
+      if (!fullResponse.trim() && outputText?.trim()) {
+        const label = toolName.replace(/_/g, ' ')
+        fullResponse = `Used **${label}** — see tool output above for details.`
+      }
       calledNames.add(toolName)
       return true
     } catch (forceErr: any) {
@@ -854,7 +892,12 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
   const heartbeatConfig = input.heartbeatConfig
   let heartbeatClassification: 'suppress' | 'strip' | 'keep' | null = null
   if (isHeartbeatRun && textForPersistence.length > 0) {
-    heartbeatClassification = classifyHeartbeatResponse(textForPersistence, heartbeatConfig?.ackMaxChars ?? 300)
+    heartbeatClassification = classifyHeartbeatResponse(textForPersistence, heartbeatConfig?.ackMaxChars ?? 300, toolEvents.length > 0)
+  }
+
+  // Emit WS notification for every heartbeat completion so UI can show pulse
+  if (isHeartbeatRun && session.agentId) {
+    notify(`heartbeat:agent:${session.agentId}`)
   }
 
   const shouldPersistAssistant = textForPersistence.length > 0
@@ -904,6 +947,7 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
         role: 'assistant',
         text: persistedText,
         time: Date.now(),
+        thinking: thinkingText || undefined,
         toolEvents: toolEvents.length ? toolEvents : undefined,
         kind: persistedKind,
       })
@@ -944,6 +988,22 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
           // Best effort — connector manager may not be loaded
         }
       }
+
+      // Auto-discover connectors linked to this agent when no explicit target is set
+      if (isHeartbeatRun && !heartbeatConfig?.target && heartbeatConfig?.showAlerts !== false && session.agentId) {
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-require-imports
+          const { listRunningConnectors: listRunning, sendConnectorMessage: sendMsg } = require('./connectors/manager')
+          const agentConnectors = listRunning().filter((c: { agentId: string | null; recentChannelId: string | null; supportsSend: boolean }) =>
+            c.agentId === session.agentId && c.recentChannelId && c.supportsSend
+          )
+          for (const conn of agentConnectors) {
+            sendMsg({ connectorId: conn.id, channelId: conn.recentChannelId, text: persistedText }).catch(() => {})
+          }
+        } catch {
+          // Best effort — connector manager may not be loaded
+        }
+      }
     }
 
     const autoMemoryEligible = shouldStoreAutoMemoryNote({

package/src/lib/server/chatroom-helpers.ts

@@ -0,0 +1,146 @@
+import { loadSettings, loadSkills, loadCredentials, decryptKey } from './storage'
+import { buildCurrentDateTimePromptContext } from './prompt-runtime-context'
+import type { Chatroom, Agent, Session, Message } from '@/types'
+
+/** Resolve API key from an agent's credentialId */
+export function resolveApiKey(credentialId: string | null | undefined): string | null {
+  if (!credentialId) return null
+  const creds = loadCredentials()
+  const cred = creds[credentialId]
+  if (!cred?.encryptedKey) return null
+  try { return decryptKey(cred.encryptedKey) } catch { return null }
+}
+
+/** Parse @mentions from message text, returns matching agentIds */
+export function parseMentions(text: string, agents: Record<string, Agent>, memberIds: string[]): string[] {
+  if (/@all\b/i.test(text)) return [...memberIds]
+  const mentionPattern = /@(\S+)/g
+  const mentioned: string[] = []
+  let match: RegExpExecArray | null
+  while ((match = mentionPattern.exec(text)) !== null) {
+    const name = match[1].toLowerCase()
+    for (const id of memberIds) {
+      const agent = agents[id]
+      if (agent && agent.name.toLowerCase().replace(/\s+/g, '') === name) {
+        if (!mentioned.includes(id)) mentioned.push(id)
+      }
+    }
+  }
+  return mentioned
+}
+
+/** Build chatroom context as a system prompt addendum with agent profiles and collaboration guidelines */
+export function buildChatroomSystemPrompt(chatroom: Chatroom, agents: Record<string, Agent>, agentId: string): string {
+  const selfAgent = agents[agentId]
+  const selfName = selfAgent?.name || agentId
+
+  // Build team profiles with capabilities
+  const teamProfiles = chatroom.agentIds
+    .filter((id) => id !== agentId)
+    .map((id) => {
+      const a = agents[id]
+      if (!a) return null
+      const tools = a.tools?.length ? `Tools: ${a.tools.join(', ')}` : 'No specialized tools'
+      const desc = a.description || a.soul || 'No description'
+      return `- **${a.name}**: ${desc}\n  ${tools}`
+    })
+    .filter(Boolean)
+    .join('\n')
+
+  const recentMessages = chatroom.messages.slice(-30).map((m) => {
+    return `[${m.senderName}]: ${m.text}`
+  }).join('\n')
+
+  const memberCount = chatroom.agentIds.length
+  const otherNames = chatroom.agentIds
+    .filter((id) => id !== agentId)
+    .map((id) => agents[id]?.name)
+    .filter(Boolean)
+
+  return [
+    `## Chatroom Context`,
+    `You are **${selfName}** in a group chatroom called "${chatroom.name}" with ${memberCount} participants (you, ${otherNames.join(', ') || 'others'}, and the user).`,
+    selfAgent?.description ? `Your role: ${selfAgent.description}` : '',
+    selfAgent?.tools?.length ? `Your available tools: ${selfAgent.tools.join(', ')}` : '',
+    '',
+    '## Team Members',
+    teamProfiles || '(no other agents)',
+    '',
+    '## How to Behave in This Chatroom',
+    '- **You are in a group chat.** Talk like you are in a real-time conversation with teammates — be direct, casual, and concise.',
+    '- **Be yourself.** Respond with personality. Don\'t give generic "let me know if you need anything" responses. Actually engage with what was said.',
+    '- **Answer the question or react to the message.** If someone says "how are you doing?" just answer naturally. If someone asks a question you can help with, help directly.',
+    '- **Keep responses short** unless depth is needed. A few sentences is usually enough. This is a chat, not an essay.',
+    '- **@mention teammates** only when you genuinely need their specific expertise. Don\'t tag people just to be polite.',
+    '- **Don\'t narrate your capabilities** unless asked. Just demonstrate them by doing things.',
+    '- **Read the room.** Look at recent messages to understand context. Don\'t repeat what others already said.',
+    '',
+    '## Recent Messages',
+    recentMessages || '(no messages yet)',
+  ].filter((line) => line !== undefined).join('\n')
+}
+
+/** Build a synthetic session object for an agent in a chatroom */
+export function buildSyntheticSession(agent: Agent, chatroomId: string): Session {
+  return {
+    id: `chatroom-${chatroomId}-${agent.id}`,
+    name: `Chatroom session for ${agent.name}`,
+    cwd: process.cwd(),
+    user: 'chatroom',
+    provider: agent.provider,
+    model: agent.model,
+    credentialId: agent.credentialId ?? null,
+    fallbackCredentialIds: agent.fallbackCredentialIds,
+    apiEndpoint: agent.apiEndpoint ?? null,
+    claudeSessionId: null,
+    messages: [],
+    createdAt: Date.now(),
+    lastActiveAt: Date.now(),
+    tools: agent.tools || [],
+    agentId: agent.id,
+  }
+}
+
+/** Build agent's system prompt including skills */
+export function buildAgentSystemPromptForChatroom(agent: Agent): string {
+  const settings = loadSettings()
+  const parts: string[] = []
+  if (settings.userPrompt) parts.push(settings.userPrompt)
+  parts.push(buildCurrentDateTimePromptContext())
+  if (agent.soul) parts.push(agent.soul)
+  if (agent.systemPrompt) parts.push(agent.systemPrompt)
+  if (agent.skillIds?.length) {
+    const allSkills = loadSkills()
+    for (const skillId of agent.skillIds) {
+      const skill = allSkills[skillId]
+      if (skill?.content) parts.push(`## Skill: ${skill.name}\n${skill.content}`)
+    }
+  }
+  return parts.join('\n\n')
+}
+
+/** Convert chatroom messages to Message history format for LLM */
+export function buildHistoryForAgent(chatroom: Chatroom, agentId: string, imagePath?: string, attachedFiles?: string[]): Message[] {
+  const history = chatroom.messages.slice(-50).map((m) => {
+    let msgText = `[${m.senderName}]: ${m.text}`
+    // Include attachment info in history
+    if (m.attachedFiles?.length) {
+      const names = m.attachedFiles.map((f) => f.split('/').pop()).join(', ')
+      msgText += `\n[Attached: ${names}]`
+    }
+    return {
+      role: m.senderId === agentId ? 'assistant' as const : 'user' as const,
+      text: msgText,
+      time: m.time,
+      ...(m.imagePath ? { imagePath: m.imagePath } : {}),
+      ...(m.attachedFiles ? { attachedFiles: m.attachedFiles } : {}),
+    }
+  })
+  // Pass through imagePath/attachedFiles from the current message to the last history entry
+  if (history.length > 0 && (imagePath || attachedFiles)) {
+    const last = history[history.length - 1]
+    if (imagePath && !last.imagePath) last.imagePath = imagePath
+    if (attachedFiles && !last.attachedFiles) last.attachedFiles = attachedFiles
+  }
+  return history
+}