@swarmclawai/swarmclaw 0.3.1 → 0.4.5

package/src/lib/server/ws-hub.ts

@@ -0,0 +1,85 @@
+import { WebSocketServer, WebSocket } from 'ws'
+import type { IncomingMessage } from 'http'
+import { validateAccessKey } from './storage'
+
+interface WsClient {
+  ws: WebSocket
+  topics: Set<string>
+}
+
+interface WsHub {
+  wss: WebSocketServer
+  clients: Set<WsClient>
+}
+
+const GK = '__swarmclaw_ws__' as const
+
+function getHub(): WsHub | null {
+  return (globalThis as any)[GK] ?? null
+}
+
+export function initWsServer() {
+  if (getHub()) return
+
+  const port = Number(process.env.WS_PORT) || (Number(process.env.PORT) || 3456) + 1
+  const wss = new WebSocketServer({ port, path: '/ws' })
+  const clients = new Set<WsClient>()
+
+  const hub: WsHub = { wss, clients }
+  ;(globalThis as any)[GK] = hub
+
+  wss.on('connection', (ws: WebSocket, req: IncomingMessage) => {
+    // Auth: validate ?key= from upgrade URL
+    const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`)
+    const key = url.searchParams.get('key') || ''
+    if (!validateAccessKey(key)) {
+      ws.close(4001, 'Unauthorized')
+      return
+    }
+
+    const client: WsClient = { ws, topics: new Set() }
+    clients.add(client)
+
+    ws.on('message', (raw) => {
+      try {
+        const msg = JSON.parse(String(raw))
+        if (msg.type === 'subscribe' && Array.isArray(msg.topics)) {
+          for (const t of msg.topics) {
+            if (typeof t === 'string') client.topics.add(t)
+          }
+        } else if (msg.type === 'unsubscribe' && Array.isArray(msg.topics)) {
+          for (const t of msg.topics) client.topics.delete(t)
+        }
+      } catch {
+        // ignore malformed messages
+      }
+    })
+
+    ws.on('close', () => {
+      clients.delete(client)
+    })
+
+    ws.on('error', () => {
+      clients.delete(client)
+    })
+  })
+
+  wss.on('error', (err) => {
+    console.error('[ws-hub] WebSocket server error:', err.message)
+  })
+
+  console.log(`[ws-hub] WebSocket server listening on port ${port}`)
+}
+
+export function notify(topic: string, action = 'update', id?: string) {
+  const hub = getHub()
+  if (!hub) return
+
+  const payload = JSON.stringify(id ? { topic, action, id } : { topic, action })
+
+  for (const client of hub.clients) {
+    if (client.topics.has(topic) && client.ws.readyState === WebSocket.OPEN) {
+      client.ws.send(payload)
+    }
+  }
+}

package/src/lib/tool-definitions.ts

@@ -0,0 +1,44 @@
+export interface ToolDefinition {
+  id: string
+  label: string
+  description: string
+}
+
+export const AVAILABLE_TOOLS: ToolDefinition[] = [
+  { id: 'shell', label: 'Shell', description: 'Execute commands in the working directory' },
+  { id: 'files', label: 'Files', description: 'Read, write, list, move, copy, and send files' },
+  { id: 'copy_file', label: 'Copy File', description: 'Copy files within the working directory' },
+  { id: 'move_file', label: 'Move File', description: 'Move/rename files within the working directory' },
+  { id: 'delete_file', label: 'Delete File', description: 'Delete files/directories (disabled by default)' },
+  { id: 'edit_file', label: 'Edit File', description: 'Search-and-replace editing within files' },
+  { id: 'process', label: 'Process', description: 'Monitor and control long-running shell commands' },
+  { id: 'web_search', label: 'Web Search', description: 'Search the web via DuckDuckGo' },
+  { id: 'web_fetch', label: 'Web Fetch', description: 'Fetch and extract text from URLs' },
+  { id: 'claude_code', label: 'Claude Code', description: 'Delegate complex tasks to Claude Code CLI' },
+  { id: 'codex_cli', label: 'Codex CLI', description: 'Delegate complex tasks to OpenAI Codex CLI' },
+  { id: 'opencode_cli', label: 'OpenCode CLI', description: 'Delegate complex tasks to OpenCode CLI' },
+  { id: 'browser', label: 'Browser', description: 'Playwright — browse, scrape, interact with web pages' },
+  { id: 'memory', label: 'Memory', description: 'Store and retrieve long-term memories across conversations' },
+  { id: 'sandbox', label: 'Sandbox', description: 'Run JS/TS/Python code in an isolated Deno sandbox' },
+  { id: 'create_document', label: 'Create Document', description: 'Render markdown to PDF, HTML, or image' },
+  { id: 'create_spreadsheet', label: 'Create Spreadsheet', description: 'Create Excel or CSV files from structured data' },
+]
+
+export const PLATFORM_TOOLS: ToolDefinition[] = [
+  { id: 'manage_agents', label: 'Agents', description: 'Create, edit, and delete agents' },
+  { id: 'manage_tasks', label: 'Tasks', description: 'Create, edit, and delete tasks' },
+  { id: 'manage_schedules', label: 'Schedules', description: 'Create, edit, and delete schedules' },
+  { id: 'manage_skills', label: 'Skills', description: 'Create, edit, and delete skills' },
+  { id: 'manage_documents', label: 'Documents', description: 'Upload, search, and delete indexed documents' },
+  { id: 'manage_webhooks', label: 'Webhooks', description: 'Register webhooks that trigger agent workflows' },
+  { id: 'manage_connectors', label: 'Connectors', description: 'Create, edit, and delete connectors' },
+  { id: 'manage_sessions', label: 'Chats', description: 'List chats, send messages, and spawn agent work' },
+  { id: 'manage_secrets', label: 'Secrets', description: 'Store and retrieve encrypted service secrets' },
+]
+
+export const ALL_TOOLS: ToolDefinition[] = [...AVAILABLE_TOOLS, ...PLATFORM_TOOLS]
+
+/** Flat id→label lookup for display */
+export const TOOL_LABELS: Record<string, string> = Object.fromEntries(
+  ALL_TOOLS.map((t) => [t.id, t.label]),
+)

package/src/lib/tts-stream.ts

@@ -0,0 +1,130 @@
+/**
+ * Streaming TTS utilities: sentence accumulation and ordered audio playback.
+ */
+
+// ---------------------------------------------------------------------------
+// SentenceAccumulator — buffers text deltas, emits on sentence boundaries
+// ---------------------------------------------------------------------------
+
+export class SentenceAccumulator {
+  private buffer = ''
+  private onSentence: (sentence: string) => void
+
+  constructor(onSentence: (sentence: string) => void) {
+    this.onSentence = onSentence
+  }
+
+  push(delta: string) {
+    this.buffer += delta
+    // Emit on sentence-ending punctuation followed by space or newline
+    const sentenceEnd = /([.!?])\s+/g
+    let match: RegExpExecArray | null
+    let lastIndex = 0
+    while ((match = sentenceEnd.exec(this.buffer)) !== null) {
+      const sentence = this.buffer.slice(lastIndex, match.index + 1).trim()
+      if (sentence) this.onSentence(sentence)
+      lastIndex = match.index + match[0].length
+    }
+    // Also emit on double newlines
+    const doubleNewline = this.buffer.indexOf('\n\n', lastIndex)
+    if (doubleNewline !== -1) {
+      const sentence = this.buffer.slice(lastIndex, doubleNewline).trim()
+      if (sentence) this.onSentence(sentence)
+      lastIndex = doubleNewline + 2
+    }
+    // Flush if buffer exceeds 200 chars without a break
+    if (this.buffer.length - lastIndex > 200) {
+      const sentence = this.buffer.slice(lastIndex).trim()
+      if (sentence) this.onSentence(sentence)
+      lastIndex = this.buffer.length
+    }
+    this.buffer = this.buffer.slice(lastIndex)
+  }
+
+  flush() {
+    const remaining = this.buffer.trim()
+    if (remaining) this.onSentence(remaining)
+    this.buffer = ''
+  }
+}
+
+// ---------------------------------------------------------------------------
+// AudioChunkQueue — ordered sequential playback of audio chunks
+// ---------------------------------------------------------------------------
+
+export class AudioChunkQueue {
+  private queue: Promise<ArrayBuffer>[] = []
+  private playing = false
+  private audioCtx: AudioContext | null = null
+  private currentSource: AudioBufferSourceNode | null = null
+  private stopped = false
+  onComplete?: () => void
+
+  enqueue(fetchPromise: Promise<ArrayBuffer>) {
+    this.queue.push(fetchPromise)
+    if (!this.playing) this.playNext()
+  }
+
+  private async playNext() {
+    if (this.stopped) return
+    if (this.queue.length === 0) {
+      this.playing = false
+      this.onComplete?.()
+      return
+    }
+
+    this.playing = true
+    const bufferPromise = this.queue.shift()!
+    try {
+      if (!this.audioCtx) this.audioCtx = new AudioContext()
+      if (this.audioCtx.state === 'suspended') await this.audioCtx.resume()
+
+      const arrayBuffer = await bufferPromise
+      if (this.stopped) return
+      const audioBuffer = await this.audioCtx.decodeAudioData(arrayBuffer)
+      if (this.stopped) return
+
+      const source = this.audioCtx.createBufferSource()
+      source.buffer = audioBuffer
+      source.connect(this.audioCtx.destination)
+      this.currentSource = source
+
+      await new Promise<void>((resolve) => {
+        source.onended = () => {
+          this.currentSource = null
+          resolve()
+        }
+        source.start()
+      })
+    } catch {
+      // Skip failed chunks
+    }
+
+    if (!this.stopped) this.playNext()
+  }
+
+  stop() {
+    this.stopped = true
+    this.queue = []
+    if (this.currentSource) {
+      try { this.currentSource.stop() } catch { /* noop */ }
+      this.currentSource = null
+    }
+    this.playing = false
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helper to fetch streaming TTS audio
+// ---------------------------------------------------------------------------
+
+export function fetchStreamTts(text: string): Promise<ArrayBuffer> {
+  return fetch('/api/tts/stream', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ text }),
+  }).then((res) => {
+    if (!res.ok) throw new Error(`TTS error: ${res.status}`)
+    return res.arrayBuffer()
+  })
+}

package/src/lib/upload.ts

@@ -1,10 +1,16 @@
 import type { UploadResult } from '../types'
+import { getStoredAccessKey } from './api-client'
 
 export async function uploadImage(file: File): Promise<UploadResult> {
+  const key = getStoredAccessKey()
   const res = await fetch('/api/upload', {
     method: 'POST',
-    headers: { 'X-Filename': file.name },
+    headers: {
+      'X-Filename': file.name,
+      ...(key ? { 'X-Access-Key': key } : {}),
+    },
     body: file,
   })
+  if (!res.ok) throw new Error(`Upload failed (${res.status})`)
   return res.json()
 }

package/src/lib/view-routes.ts

@@ -0,0 +1,28 @@
+import type { AppView } from '@/types'
+
+export const DEFAULT_VIEW: AppView = 'agents'
+
+export const VIEW_TO_PATH: Record<AppView, string> = {
+  agents: '/agents',
+  schedules: '/schedules',
+  memory: '/memory',
+  tasks: '/tasks',
+  secrets: '/secrets',
+  providers: '/providers',
+  skills: '/skills',
+  connectors: '/connectors',
+  webhooks: '/webhooks',
+  mcp_servers: '/mcp-servers',
+  knowledge: '/knowledge',
+  plugins: '/plugins',
+  usage: '/usage',
+  runs: '/runs',
+  logs: '/logs',
+  settings: '/settings',
+  projects: '/projects',
+}
+
+const entries = Object.entries(VIEW_TO_PATH) as [AppView, string][]
+export const PATH_TO_VIEW: Record<string, AppView> = Object.fromEntries(
+  entries.map(([view, path]) => [path, view]),
+) as Record<string, AppView>

package/src/lib/ws-client.ts

@@ -0,0 +1,124 @@
+type WsCallback = () => void
+
+let ws: WebSocket | null = null
+let accessKey = ''
+let reconnectTimer: ReturnType<typeof setTimeout> | null = null
+let reconnectDelay = 1000
+const MAX_RECONNECT_DELAY = 30_000
+const listeners = new Map<string, Set<WsCallback>>()
+let connected = false
+
+function getWsUrl(key: string): string {
+  const host = typeof window !== 'undefined' ? window.location.hostname : 'localhost'
+  const port = process.env.NEXT_PUBLIC_WS_PORT || '3457'
+  const protocol = typeof window !== 'undefined' && window.location.protocol === 'https:' ? 'wss' : 'ws'
+  return `${protocol}://${host}:${port}/ws?key=${encodeURIComponent(key)}`
+}
+
+function handleMessage(event: MessageEvent) {
+  try {
+    const msg = JSON.parse(event.data)
+    const topic = msg.topic as string
+    if (!topic) return
+    const cbs = listeners.get(topic)
+    if (cbs) {
+      for (const cb of cbs) cb()
+    }
+  } catch {
+    // ignore malformed
+  }
+}
+
+function scheduleReconnect() {
+  if (reconnectTimer) return
+  reconnectTimer = setTimeout(() => {
+    reconnectTimer = null
+    if (!accessKey) return
+    connect(accessKey)
+  }, reconnectDelay)
+  reconnectDelay = Math.min(reconnectDelay * 2, MAX_RECONNECT_DELAY)
+}
+
+function connect(key: string) {
+  if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) return
+
+  try {
+    ws = new WebSocket(getWsUrl(key))
+  } catch {
+    scheduleReconnect()
+    return
+  }
+
+  ws.onopen = () => {
+    connected = true
+    reconnectDelay = 1000
+    // Subscribe to all currently registered topics
+    const topics = Array.from(listeners.keys())
+    if (topics.length > 0) {
+      ws?.send(JSON.stringify({ type: 'subscribe', topics }))
+    }
+  }
+
+  ws.onmessage = handleMessage
+
+  ws.onclose = () => {
+    connected = false
+    ws = null
+    if (accessKey) scheduleReconnect()
+  }
+
+  ws.onerror = () => {
+    // onclose will fire after this
+  }
+}
+
+export function connectWs(key: string) {
+  accessKey = key
+  reconnectDelay = 1000
+  connect(key)
+}
+
+export function disconnectWs() {
+  accessKey = ''
+  if (reconnectTimer) {
+    clearTimeout(reconnectTimer)
+    reconnectTimer = null
+  }
+  if (ws) {
+    ws.onclose = null
+    ws.close()
+    ws = null
+  }
+  connected = false
+}
+
+export function subscribeWs(topic: string, callback: WsCallback) {
+  let set = listeners.get(topic)
+  const isNew = !set
+  if (!set) {
+    set = new Set()
+    listeners.set(topic, set)
+  }
+  set.add(callback)
+
+  // Tell server about new topic subscription
+  if (isNew && ws?.readyState === WebSocket.OPEN) {
+    ws.send(JSON.stringify({ type: 'subscribe', topics: [topic] }))
+  }
+}
+
+export function unsubscribeWs(topic: string, callback: WsCallback) {
+  const set = listeners.get(topic)
+  if (!set) return
+  set.delete(callback)
+  if (set.size === 0) {
+    listeners.delete(topic)
+    if (ws?.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify({ type: 'unsubscribe', topics: [topic] }))
+    }
+  }
+}
+
+export function isWsConnected(): boolean {
+  return connected
+}

package/src/proxy.ts

@@ -9,6 +9,8 @@ export function proxy(request: NextRequest) {
   const { pathname } = request.nextUrl
   const isWebhookTrigger = request.method === 'POST'
     && /^\/api\/webhooks\/[^/]+\/?$/.test(pathname)
+  const isConnectorWebhook = request.method === 'POST'
+    && /^\/api\/connectors\/[^/]+\/webhook\/?$/.test(pathname)
 
   // Only protect API routes (not auth, uploads served as static assets, or inbound webhooks)
   if (
@@ -16,6 +18,7 @@ export function proxy(request: NextRequest) {
     || pathname === '/api/auth'
     || pathname.startsWith('/api/uploads/')
     || isWebhookTrigger
+    || isConnectorWebhook
   ) {
     return NextResponse.next()
   }

package/src/stores/use-app-store.ts

@@ -1,7 +1,7 @@
 'use client'
 
 import { create } from 'zustand'
-import type { Sessions, Session, NetworkInfo, Directory, ProviderInfo, Credentials, Agent, Schedule, AppView, BoardTask, AppSettings, OrchestratorSecret, ProviderConfig, Skill, Connector, Webhook, McpServerConfig, PluginMeta } from '../types'
+import type { Sessions, Session, NetworkInfo, Directory, ProviderInfo, Credentials, Agent, Schedule, AppView, BoardTask, AppSettings, OrchestratorSecret, ProviderConfig, Skill, Connector, Webhook, McpServerConfig, PluginMeta, Project } from '../types'
 import { fetchSessions, fetchDirs, fetchProviders, fetchCredentials } from '../lib/sessions'
 import { fetchAgents } from '../lib/agents'
 import { fetchSchedules } from '../lib/schedules'
@@ -151,6 +151,16 @@ interface AppState {
   editingPluginFilename: string | null
   setEditingPluginFilename: (filename: string | null) => void
 
+  // Projects
+  projects: Record<string, Project>
+  loadProjects: () => Promise<void>
+  projectSheetOpen: boolean
+  setProjectSheetOpen: (open: boolean) => void
+  editingProjectId: string | null
+  setEditingProjectId: (id: string | null) => void
+  activeProjectFilter: string | null
+  setActiveProjectFilter: (id: string | null) => void
+
 }
 
 export const useAppStore = create<AppState>((set, get) => ({
@@ -465,4 +475,21 @@ export const useAppStore = create<AppState>((set, get) => ({
   editingPluginFilename: null,
   setEditingPluginFilename: (filename) => set({ editingPluginFilename: filename }),
 
+  // Projects
+  projects: {},
+  loadProjects: async () => {
+    try {
+      const projects = await api<Record<string, Project>>('GET', '/projects')
+      set({ projects })
+    } catch {
+      // ignore
+    }
+  },
+  projectSheetOpen: false,
+  setProjectSheetOpen: (open) => set({ projectSheetOpen: open }),
+  editingProjectId: null,
+  setEditingProjectId: (id) => set({ editingProjectId: id }),
+  activeProjectFilter: null,
+  setActiveProjectFilter: (id) => set({ activeProjectFilter: id }),
+
 }))

package/src/stores/use-chat-store.ts

@@ -7,7 +7,7 @@ import { speak } from '../lib/tts'
 import { getStoredAccessKey } from '../lib/api-client'
 import { useAppStore } from './use-app-store'
 
-interface PendingImage {
+export interface PendingFile {
   file: File
   path: string
   url: string
@@ -44,8 +44,15 @@ interface ChatState {
   ttsEnabled: boolean
   toggleTts: () => void
 
-  pendingImage: PendingImage | null
-  setPendingImage: (img: PendingImage | null) => void
+  // Multi-file attachment support
+  pendingFiles: PendingFile[]
+  addPendingFile: (f: PendingFile) => void
+  removePendingFile: (index: number) => void
+  clearPendingFiles: () => void
+
+  // Legacy single-image compat (reads first pendingFile)
+  pendingImage: PendingFile | null
+  setPendingImage: (img: PendingFile | null) => void
 
   devServer: DevServerStatus | null
   setDevServer: (ds: DevServerStatus | null) => void
@@ -57,6 +64,10 @@ interface ChatState {
   retryLastMessage: () => Promise<void>
   sendHeartbeat: (sessionId: string) => Promise<void>
   stopStreaming: () => void
+
+  // Voice conversation
+  voiceConversationActive: boolean
+  onStreamEvent: ((event: { t: string; text?: string }) => void) | null
 }
 
 export const useChatStore = create<ChatState>((set, get) => ({
@@ -70,21 +81,36 @@ export const useChatStore = create<ChatState>((set, get) => ({
   lastUsage: null,
   ttsEnabled: false,
   toggleTts: () => set((s) => ({ ttsEnabled: !s.ttsEnabled })),
-  pendingImage: null,
-  setPendingImage: (img) => set({ pendingImage: img }),
+  voiceConversationActive: false,
+  onStreamEvent: null,
+
+  pendingFiles: [],
+  addPendingFile: (f) => set((s) => ({ pendingFiles: [...s.pendingFiles, f] })),
+  removePendingFile: (index) => set((s) => ({ pendingFiles: s.pendingFiles.filter((_, i) => i !== index) })),
+  clearPendingFiles: () => set({ pendingFiles: [] }),
+
+  // Legacy compat: pendingImage reads/writes the first pending file
+  get pendingImage() { const files = get().pendingFiles; return files.length ? files[0] : null },
+  setPendingImage: (img) => set({ pendingFiles: img ? [img] : [] }),
+
   devServer: null,
   setDevServer: (ds) => set({ devServer: ds }),
   debugOpen: false,
   setDebugOpen: (open) => set({ debugOpen: open }),
 
   sendMessage: async (text: string) => {
-    if (!text.trim() || get().streaming) return
+    const { pendingFiles } = get()
+    if ((!text.trim() && !pendingFiles.length) || get().streaming) return
     const sessionId = useAppStore.getState().currentSessionId
     if (!sessionId) return
 
-    const { pendingImage } = get()
-    const imagePath = pendingImage?.path
-    const imageUrl = pendingImage?.url
+    // Primary image (backward compat)
+    const imagePath = pendingFiles[0]?.path
+    const imageUrl = pendingFiles[0]?.url
+    // All attached file paths
+    const attachedFiles = pendingFiles.length > 1
+      ? pendingFiles.map((f) => f.path)
+      : undefined
 
     const userMsg: Message = {
       role: 'user',
@@ -92,13 +118,14 @@ export const useChatStore = create<ChatState>((set, get) => ({
       time: Date.now(),
       imagePath,
       imageUrl,
+      attachedFiles,
     }
     set((s) => ({
       streaming: true,
       streamingSessionId: sessionId,
       streamText: '',
       messages: [...s.messages, userMsg],
-      pendingImage: null,
+      pendingFiles: [],
       toolEvents: [],
       lastUsage: null,
     }))
@@ -114,6 +141,8 @@ export const useChatStore = create<ChatState>((set, get) => ({
       /cancelled by steer mode|stopped by user/i.test(msg || '')
 
     await streamChat(sessionId, text, imagePath, imageUrl, (event: SSEEvent) => {
+      // Forward events to voice conversation handler if active
+      get().onStreamEvent?.(event)
       if (event.t === 'd') {
         fullText += event.text || ''
         set({ streamText: fullText })
@@ -166,7 +195,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
       } else if (event.t === 'done') {
         // done
       }
-    })
+    }, attachedFiles)
 
     if (fullText.trim()) {
       const currentToolEvents = get().toolEvents
@@ -188,7 +217,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
         streamingSessionId: null,
         streamText: '',
       }))
-      if (get().ttsEnabled) speak(fullText)
+      if (get().ttsEnabled && !get().voiceConversationActive) speak(fullText)
     } else {
       set({ streaming: false, streamingSessionId: null, streamText: '' })
     }
@@ -218,9 +247,8 @@ export const useChatStore = create<ChatState>((set, get) => ({
         set({ messages: msgs })
       }
       // Re-send the last user message through the normal SSE flow
-      // Temporarily set pendingImage if there was one
       if (imagePath) {
-        set({ pendingImage: { file: new File([], ''), path: imagePath, url: '' } })
+        set({ pendingFiles: [{ file: new File([], ''), path: imagePath, url: '' }] })
       }
       await get().sendMessage(message)
     } catch {