crawd 0.8.0

package/src/daemon/manager.ts

@@ -0,0 +1,280 @@
+import { spawn, type ChildProcess } from 'child_process'
+import {
+  existsSync,
+  mkdirSync,
+  createWriteStream,
+  cpSync,
+  writeFileSync,
+} from 'fs'
+import { join } from 'path'
+import {
+  LOGS_DIR,
+  LOG_FILES,
+  OVERLAY_DIR,
+  BACKEND_DIR,
+  BACKEND_TEMPLATE_DIR,
+  CRAWD_HOME,
+  TTS_CACHE_DIR,
+} from '../utils/paths.js'
+import { log } from '../utils/logger.js'
+import { writePid, killProcess, isRunning, type ProcessName } from './pid.js'
+import { loadConfig, loadEnv } from '../config/store.js'
+
+/** Ensure all required directories exist */
+export function ensureDirectories() {
+  const dirs = [CRAWD_HOME, LOGS_DIR, OVERLAY_DIR, BACKEND_DIR, TTS_CACHE_DIR]
+  for (const dir of dirs) {
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true })
+    }
+  }
+}
+
+/** Copy overlay template to user directory if needed */
+export function ensureOverlay(force = false) {
+  if (force || !existsSync(join(OVERLAY_DIR, 'package.json'))) {
+    log.info('Setting up overlay...')
+
+    if (!existsSync(OVERLAY_TEMPLATE_DIR)) {
+      throw new Error(
+        `Overlay template not found at ${OVERLAY_TEMPLATE_DIR}. ` +
+        'This is a bug in the crawd package.'
+      )
+    }
+
+    cpSync(OVERLAY_TEMPLATE_DIR, OVERLAY_DIR, { recursive: true })
+    log.success('Overlay installed to ' + OVERLAY_DIR)
+    return true
+  }
+  return false
+}
+
+/** Copy backend template to user directory if needed */
+export function ensureBackend(force = false) {
+  if (force || !existsSync(join(BACKEND_DIR, 'index.ts'))) {
+    log.info('Setting up backend...')
+
+    if (!existsSync(BACKEND_TEMPLATE_DIR)) {
+      throw new Error(
+        `Backend template not found at ${BACKEND_TEMPLATE_DIR}. ` +
+        'This is a bug in the crawd package.'
+      )
+    }
+
+    cpSync(BACKEND_TEMPLATE_DIR, BACKEND_DIR, { recursive: true })
+
+    // Create a minimal package.json for the backend
+    const backendPkg = {
+      name: '@crawd/backend',
+      type: 'module',
+      private: true,
+      dependencies: {
+        '@fastify/cors': '^10.0.2',
+        '@fastify/static': '^8.1.0',
+        'dotenv': '^16.4.7',
+        'fastify': '^5.2.1',
+        'socket.io': '^4.8.1',
+      },
+      optionalDependencies: {
+        'openai': '^4.77.0',
+        '@elevenlabs/elevenlabs-js': '^1.0.0',
+      },
+    }
+
+    writeFileSync(
+      join(BACKEND_DIR, 'package.json'),
+      JSON.stringify(backendPkg, null, 2)
+    )
+
+    log.success('Backend installed to ' + BACKEND_DIR)
+    return true
+  }
+  return false
+}
+
+/** Check if backend dependencies are installed */
+function backendHasNodeModules(): boolean {
+  return existsSync(join(BACKEND_DIR, 'node_modules'))
+}
+
+/** Install backend dependencies */
+async function installBackendDeps(): Promise<void> {
+  log.info('Installing backend dependencies...')
+
+  return new Promise((resolve, reject) => {
+    const child = spawn('pnpm', ['install'], {
+      cwd: BACKEND_DIR,
+      stdio: 'inherit',
+    })
+
+    child.on('close', (code) => {
+      if (code === 0) {
+        log.success('Backend dependencies installed')
+        resolve()
+      } else {
+        reject(new Error(`pnpm install failed with code ${code}`))
+      }
+    })
+
+    child.on('error', reject)
+  })
+}
+
+/** Check if overlay dependencies are installed */
+function overlayHasNodeModules(): boolean {
+  return existsSync(join(OVERLAY_DIR, 'node_modules'))
+}
+
+/** Install overlay dependencies */
+async function installOverlayDeps(): Promise<void> {
+  log.info('Installing overlay dependencies...')
+
+  return new Promise((resolve, reject) => {
+    const child = spawn('pnpm', ['install'], {
+      cwd: OVERLAY_DIR,
+      stdio: 'inherit',
+    })
+
+    child.on('close', (code) => {
+      if (code === 0) {
+        log.success('Overlay dependencies installed')
+        resolve()
+      } else {
+        reject(new Error(`pnpm install failed with code ${code}`))
+      }
+    })
+
+    child.on('error', reject)
+  })
+}
+
+/** Build environment variables for child processes */
+function buildEnv(): NodeJS.ProcessEnv {
+  const config = loadConfig()
+  const secrets = loadEnv()
+  const env: NodeJS.ProcessEnv = { ...process.env }
+
+  // Secrets from ~/.crawd/.env
+  for (const [key, value] of Object.entries(secrets)) {
+    env[key] = value
+  }
+
+  // Config from ~/.crawd/config.json
+  env.CRAWD_GATEWAY_URL = config.gateway.url
+  env.CRAWD_CHANNEL_ID = config.gateway.channelId
+  env.CRAWD_BACKEND_PORT = String(config.ports.backend)
+  env.CRAWD_OVERLAY_PORT = String(config.ports.overlay)
+
+  return env
+}
+
+/** Start a daemon process */
+function startDaemon(
+  name: ProcessName,
+  command: string,
+  args: string[],
+  cwd: string,
+  env: NodeJS.ProcessEnv
+): ChildProcess {
+  // Ensure log directory exists
+  if (!existsSync(LOGS_DIR)) {
+    mkdirSync(LOGS_DIR, { recursive: true })
+  }
+
+  const logStream = createWriteStream(LOG_FILES[name], { flags: 'a' })
+
+  const child = spawn(command, args, {
+    cwd,
+    env,
+    detached: true,
+    stdio: ['ignore', logStream, logStream],
+  })
+
+  child.unref()
+
+  if (child.pid) {
+    writePid(name, child.pid)
+  }
+
+  return child
+}
+
+/** Start the backend server */
+export async function startBackend(): Promise<ChildProcess | null> {
+  if (isRunning('backend')) {
+    log.warn('Backend is already running')
+    return null
+  }
+
+  // Ensure backend is set up
+  ensureBackend()
+
+  // Install dependencies if needed
+  if (!backendHasNodeModules()) {
+    await installBackendDeps()
+  }
+
+  const config = loadConfig()
+  const env = buildEnv()
+
+  // Set TTS directory
+  env.CRAWD_TTS_DIR = TTS_CACHE_DIR
+
+  const backendEntry = join(BACKEND_DIR, 'index.ts')
+
+  log.info(`Starting backend on port ${config.ports.backend}...`)
+  return startDaemon('backend', 'npx', ['tsx', backendEntry], BACKEND_DIR, env)
+}
+
+/** Start the overlay dev server */
+export async function startOverlay(): Promise<ChildProcess | null> {
+  if (isRunning('overlay')) {
+    log.warn('Overlay is already running')
+    return null
+  }
+
+  ensureOverlay()
+
+  if (!overlayHasNodeModules()) {
+    await installOverlayDeps()
+  }
+
+  const config = loadConfig()
+  const env = buildEnv()
+
+  // Set Vite port
+  env.PORT = String(config.ports.overlay)
+
+  log.info(`Starting overlay on port ${config.ports.overlay}...`)
+  return startDaemon('overlay', 'pnpm', ['dev'], OVERLAY_DIR, env)
+}
+
+/** Stop a specific daemon */
+export function stopDaemon(name: ProcessName): boolean {
+  if (!isRunning(name)) {
+    log.dim(`${name} is not running`)
+    return false
+  }
+
+  const killed = killProcess(name)
+  if (killed) {
+    log.success(`Stopped ${name}`)
+  }
+  return killed
+}
+
+/** Stop all daemons */
+export function stopAll() {
+  stopDaemon('backend')
+  stopDaemon('overlay')
+}
+
+/** Start all daemons */
+export async function startAll() {
+  ensureDirectories()
+
+  const backend = await startBackend()
+  const overlay = await startOverlay()
+
+  return { backend, overlay }
+}

package/src/daemon/pid.ts

@@ -0,0 +1,102 @@
+import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs'
+import { dirname } from 'path'
+import { PID_FILES, PIDS_DIR } from '../utils/paths.js'
+
+export type ProcessName = 'backend' | 'overlay' | 'crawdbot'
+
+/** Write a PID file */
+export function writePid(name: ProcessName, pid: number) {
+  if (!existsSync(PIDS_DIR)) {
+    mkdirSync(PIDS_DIR, { recursive: true })
+  }
+  writeFileSync(PID_FILES[name], String(pid))
+}
+
+/** Read a PID from file */
+export function readPid(name: ProcessName): number | null {
+  const path = PID_FILES[name]
+  if (!existsSync(path)) {
+    return null
+  }
+  try {
+    const content = readFileSync(path, 'utf-8').trim()
+    const pid = parseInt(content, 10)
+    return isNaN(pid) ? null : pid
+  } catch {
+    return null
+  }
+}
+
+/** Remove a PID file */
+export function removePid(name: ProcessName) {
+  const path = PID_FILES[name]
+  if (existsSync(path)) {
+    unlinkSync(path)
+  }
+}
+
+/** Check if a process is running by PID */
+export function isProcessRunning(pid: number): boolean {
+  try {
+    // kill with signal 0 doesn't kill the process, just checks if it exists
+    process.kill(pid, 0)
+    return true
+  } catch {
+    return false
+  }
+}
+
+/** Check if a named process is running */
+export function isRunning(name: ProcessName): boolean {
+  const pid = readPid(name)
+  if (pid === null) return false
+  return isProcessRunning(pid)
+}
+
+/** Kill a process by name if running */
+export function killProcess(name: ProcessName): boolean {
+  const pid = readPid(name)
+  if (pid === null) return false
+
+  if (!isProcessRunning(pid)) {
+    removePid(name)
+    return false
+  }
+
+  try {
+    process.kill(pid, 'SIGTERM')
+    // Give it a moment to terminate gracefully
+    setTimeout(() => {
+      if (isProcessRunning(pid)) {
+        try {
+          process.kill(pid, 'SIGKILL')
+        } catch {
+          // Process may have already exited
+        }
+      }
+    }, 2000)
+    removePid(name)
+    return true
+  } catch {
+    removePid(name)
+    return false
+  }
+}
+
+/** Get status of all processes */
+export function getProcessStatus(): Record<ProcessName, { running: boolean; pid: number | null }> {
+  return {
+    backend: {
+      running: isRunning('backend'),
+      pid: readPid('backend'),
+    },
+    overlay: {
+      running: isRunning('overlay'),
+      pid: readPid('overlay'),
+    },
+    crawdbot: {
+      running: isRunning('crawdbot'),
+      pid: readPid('crawdbot'),
+    },
+  }
+}

package/src/lib/chat/base.ts

@@ -0,0 +1,13 @@
+import { EventEmitter } from 'events'
+import type { ChatPlatform } from './types'
+
+/**
+ * Abstract base class for chat clients.
+ * Uses EventEmitter - listen with: client.on('message', (msg) => ...)
+ */
+export abstract class BaseChatClient extends EventEmitter {
+  abstract readonly platform: ChatPlatform
+  abstract connect(): Promise<void>
+  abstract disconnect(): void
+  abstract isConnected(): boolean
+}

package/src/lib/chat/manager.ts

@@ -0,0 +1,105 @@
+import type { ChatMessage } from './types'
+import type { BaseChatClient } from './base'
+
+/**
+ * Orchestrates multiple chat clients with reconnection policy.
+ * Uses string keys to allow multiple clients per platform.
+ */
+export class ChatManager {
+  private clients = new Map<string, BaseChatClient>()
+  private messageHandlers: ((msg: ChatMessage) => void)[] = []
+  private reconnectTimers = new Map<string, NodeJS.Timeout>()
+
+  /**
+   * Register a client with a unique key.
+   * Key can be platform name or custom (e.g., 'youtube:stream1')
+   */
+  registerClient(key: string, client: BaseChatClient): void {
+    this.clients.set(key, client)
+
+    client.on('message', (msg: ChatMessage) => {
+      this.messageHandlers.forEach(handler => handler(msg))
+    })
+
+    client.on('connected', () => {
+      console.log(`[ChatManager] ${key} connected`)
+      this.clearReconnectTimer(key)
+    })
+
+    client.on('disconnected', () => {
+      console.log(`[ChatManager] ${key} disconnected`)
+      this.scheduleReconnect(key)
+    })
+
+    client.on('error', (error: Error) => {
+      console.error(`[ChatManager] ${key} error:`, error.message)
+    })
+  }
+
+  private scheduleReconnect(key: string, attempt = 1): void {
+    const maxAttempts = 5
+    if (attempt > maxAttempts) {
+      console.error(`[ChatManager] ${key} failed after ${maxAttempts} attempts, giving up`)
+      return
+    }
+
+    const delay = Math.min(5000 * Math.pow(2, attempt - 1), 60000)
+    console.log(`[ChatManager] ${key} reconnecting in ${delay}ms (attempt ${attempt})`)
+
+    const timer = setTimeout(async () => {
+      const client = this.clients.get(key)
+      if (!client || client.isConnected()) return
+
+      try {
+        await client.connect()
+      } catch {
+        this.scheduleReconnect(key, attempt + 1)
+      }
+    }, delay)
+
+    this.reconnectTimers.set(key, timer)
+  }
+
+  private clearReconnectTimer(key: string): void {
+    const timer = this.reconnectTimers.get(key)
+    if (timer) {
+      clearTimeout(timer)
+      this.reconnectTimers.delete(key)
+    }
+  }
+
+  async connectAll(): Promise<void> {
+    await Promise.allSettled(
+      Array.from(this.clients.entries()).map(async ([key, client]) => {
+        try {
+          await client.connect()
+        } catch (err) {
+          console.error(`[ChatManager] ${key} initial connect failed:`, err)
+          this.scheduleReconnect(key)
+        }
+      })
+    )
+  }
+
+  disconnectAll(): void {
+    this.reconnectTimers.forEach(timer => clearTimeout(timer))
+    this.reconnectTimers.clear()
+    this.clients.forEach(client => {
+      if (client.isConnected()) client.disconnect()
+    })
+  }
+
+  onMessage(handler: (msg: ChatMessage) => void): void {
+    this.messageHandlers.push(handler)
+  }
+
+  getConnectedKeys(): string[] {
+    return Array.from(this.clients.entries())
+      .filter(([_, client]) => client.isConnected())
+      .map(([key]) => key)
+  }
+
+  getClient(key: string): BaseChatClient | undefined {
+    return this.clients.get(key)
+  }
+}

package/src/lib/chat/pumpfun/client.ts

@@ -0,0 +1,56 @@
+import { PumpfunSocketClient } from '../../pumpfun/v2/socket/client'
+import type { PumpfunMessage } from '../../pumpfun/v2/socket/types'
+import { BaseChatClient } from '../base'
+import { generateShortId } from '../types'
+import type { ChatPlatform, ChatMessage } from '../types'
+
+/**
+ * Adapter wrapping existing PumpfunSocketClient to unified interface.
+ */
+export class PumpFunChatClient extends BaseChatClient {
+  readonly platform: ChatPlatform = 'pumpfun'
+
+  private client: PumpfunSocketClient
+  private roomId: string
+  private _connected = false
+
+  constructor(roomId: string, authToken: string | null) {
+    super()
+    this.roomId = roomId
+    this.client = new PumpfunSocketClient(authToken)
+  }
+
+  async connect(): Promise<void> {
+    this.client.connect()
+    await this.client.joinRoom(this.roomId)
+
+    this.client.onMessage((msg: PumpfunMessage) => {
+      const chatMsg: ChatMessage = {
+        id: `pf:${msg.id}`,
+        shortId: generateShortId(),
+        platform: 'pumpfun',
+        username: msg.username,
+        message: msg.message,
+        timestamp: Date.now(),
+        metadata: {
+          userAddress: msg.userAddress,
+          roomId: msg.roomId
+        }
+      }
+      this.emit('message', chatMsg)
+    })
+
+    this._connected = true
+    this.emit('connected')
+  }
+
+  disconnect(): void {
+    this.client.disconnect()
+    this._connected = false
+    this.emit('disconnected')
+  }
+
+  isConnected(): boolean {
+    return this._connected
+  }
+}

package/src/lib/chat/types.ts

@@ -0,0 +1,48 @@
+import { nanoid } from 'nanoid'
+
+export type ChatPlatform = 'pumpfun' | 'youtube' | 'twitch' | 'twitter'
+
+/** Generate a short ID for chat messages (6 chars) */
+export const generateShortId = () => nanoid(6)
+
+export type SuperChatInfo = {
+  amountDisplayString: string
+  backgroundColor: string
+}
+
+export type ChatMessageMetadata = {
+  // YouTube
+  channelId?: string
+  isModerator?: boolean
+  isMember?: boolean
+  superChat?: SuperChatInfo
+
+  // Pump Fun
+  userAddress?: string
+  roomId?: string
+
+  // Common
+  avatarUrl?: string
+}
+
+/**
+ * Unified message type for all chat platforms.
+ * `platform` is optional for backward compatibility.
+ */
+export type ChatMessage = {
+  id: string
+  /** Short ID for agent prompt/response (6 chars, alphanumeric) */
+  shortId: string
+  username: string
+  message: string
+  platform?: ChatPlatform
+  timestamp?: number
+  metadata?: ChatMessageMetadata
+}
+
+export type ChatClientEvents = {
+  message: (msg: ChatMessage) => void
+  connected: () => void
+  disconnected: () => void
+  error: (error: Error) => void
+}