@luckydraw/cumulus 0.27.40 → 0.28.0

package/dist/gateway/server.js

@@ -10,9 +10,12 @@
  *   GET  /api/threads               — list all threads
  *   DELETE /api/thread/:name        — delete thread (requires X-Confirm: delete)
  *   POST /api/migrate               — sync thread + clone repo + set projectDir
+ *   POST /api/media/upload          — upload file to media directory (returns URL)
+ *   GET  /media/*                   — serve uploaded media files (no auth)
  *   GET  /health                    — health check (no auth)
  */
 import { execFileSync } from 'child_process';
+import * as crypto from 'crypto';
 import * as fs from 'fs';
 import * as http from 'http';
 import * as os from 'os';
@@ -21,11 +24,12 @@ import { WebSocketServer } from 'ws';
 import { loadThreadConfig, saveThreadConfig } from '../lib/config.js';
 import { loadEmbeddings, generateMissingEmbeddings } from '../lib/embeddings.js';
 import { ExecutorRegistry } from '../lib/executor.js';
-import { sendMessage, getOrCreateThread, clearThreadCache, } from '../lib/gateway.js';
+import { sendMessage, getOrCreateThread, clearThreadCache, getActiveSubprocessCount, } from '../lib/gateway.js';
 import { catchUpSegmentation, loadSegmentBoundaries, saveSegmentBoundaries, } from '../lib/segments.js';
 import { listTemplates, scaffoldFromTemplate } from '../lib/templates.js';
-import { serveStaticFile } from './adapters/webchat.js';
+import { serveStaticFile, getMediaDir } from './adapters/webchat.js';
 import { authenticate } from './auth.js';
+import { handleEmailHook, handleFormHook, handleGenericWebhook, } from './hooks.js';
 import { configureVapid, sendNotification, subscribe as pushSubscribe, unsubscribe as pushUnsubscribe, } from './push.js';
 const threadQueues = new Map();
 const threadBusy = new Map();
@@ -88,6 +92,7 @@ async function processMessage(threadName, item, opts) {
             extraMcpServers: opts.extraMcpServers,
             gatewayAgentsConfig: opts.gatewayAgentsConfig,
             executorRegistry: opts.executorRegistry,
+            hfApiKey: opts.hfApiKey,
             signal: ac.signal,
             onToken: text => sendSSE('token', { text }),
             onSegment: segment => sendSSE('segment', segment),
@@ -113,6 +118,83 @@ async function processMessage(threadName, item, opts) {
         }
     }
 }
+// ─── Webhook endpoints ──────────────────────────────────────────────────────
+async function handleWebhook(type, name, req, res, pipelineOpts, hooksConfig) {
+    if (!hooksConfig) {
+        jsonResponse(res, 404, { error: 'Webhooks not configured' });
+        return;
+    }
+    let body;
+    try {
+        body = await readBody(req);
+    }
+    catch {
+        jsonResponse(res, 400, { error: 'Could not read request body' });
+        return;
+    }
+    const hookOpts = {
+        basePath: pipelineOpts.basePath,
+        claudePath: pipelineOpts.claudePath,
+        sharedMcpPort: pipelineOpts.sharedMcpPort,
+        projectRoot: pipelineOpts.projectRoot,
+        extraMcpServers: pipelineOpts.extraMcpServers,
+        gatewayAgentsConfig: pipelineOpts.gatewayAgentsConfig,
+        executorRegistry: pipelineOpts.executorRegistry,
+        broadcastToThread: pipelineOpts.broadcastToThread,
+    };
+    try {
+        switch (type) {
+            case 'email': {
+                if (!hooksConfig.email?.enabled) {
+                    jsonResponse(res, 404, { error: 'Email hooks not enabled' });
+                    return;
+                }
+                const result = await handleEmailHook(body, req.headers, hooksConfig.email, hookOpts);
+                // Return 200 immediately — processing continues async
+                jsonResponse(res, 200, result);
+                return;
+            }
+            case 'form': {
+                if (!hooksConfig.form?.enabled) {
+                    jsonResponse(res, 404, { error: 'Form hooks not enabled' });
+                    return;
+                }
+                const result = await handleFormHook(body, hooksConfig.form, hookOpts);
+                jsonResponse(res, 200, result);
+                return;
+            }
+            case 'webhook': {
+                if (!name) {
+                    jsonResponse(res, 400, { error: 'Missing webhook name: POST /api/hooks/webhook/:name' });
+                    return;
+                }
+                const webhookConfig = hooksConfig.webhooks?.[name];
+                if (!webhookConfig) {
+                    jsonResponse(res, 404, { error: `Webhook "${name}" not configured` });
+                    return;
+                }
+                const result = await handleGenericWebhook(name, body, webhookConfig, hookOpts);
+                jsonResponse(res, 200, result);
+                return;
+            }
+            default:
+                jsonResponse(res, 404, { error: `Unknown hook type: ${type}` });
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        if (message.includes('Invalid webhook signature')) {
+            jsonResponse(res, 401, { error: 'Invalid webhook signature' });
+        }
+        else if (message.includes('Invalid JSON') || message.includes('Unexpected token')) {
+            jsonResponse(res, 400, { error: 'Invalid JSON body' });
+        }
+        else {
+            console.error(`[Gateway] Webhook error (${type}):`, err);
+            jsonResponse(res, 500, { error: 'Webhook processing error' });
+        }
+    }
+}
 // ─── Agent endpoints ─────────────────────────────────────────────────────────
 async function handleListAgents(res) {
     try {
@@ -153,16 +235,11 @@ async function handleAgentInject(req, res, pipelineOpts) {
     // Resolve target thread names
     let targetNames;
     if (targets === '__broadcast__') {
-        // Broadcast to all threads except sender
-        if (!fs.existsSync(THREADS_DIR)) {
-            jsonResponse(res, 200, { results: [] });
-            return;
-        }
-        targetNames = fs
-            .readdirSync(THREADS_DIR)
-            .filter(f => f.endsWith('.jsonl'))
-            .map(f => f.replace(/\.jsonl$/, ''))
-            .filter(name => name !== sender);
+        // Broadcast-all is disabled — agents must target specific threads
+        jsonResponse(res, 400, {
+            error: 'Broadcast to all agents is disabled. Use send_to_agent with specific targets.',
+        });
+        return;
     }
     else if (Array.isArray(targets)) {
         targetNames = targets;
@@ -202,7 +279,7 @@ async function handleAgentInject(req, res, pipelineOpts) {
                 visibility,
             });
             // Ensure thread exists
-            await getOrCreateThread(targetName, pipelineOpts.basePath);
+            await getOrCreateThread(targetName);
             // Send as a message through the pipeline (fire-and-forget)
             threadBusy.set(targetName, true);
             const broadcast = pipelineOpts.broadcastToThread;
@@ -392,6 +469,18 @@ function parseRoute(url) {
             query,
         };
     }
+    // Match /api/hooks/:type or /api/hooks/:type/:name
+    const hooksMatch = pathPart.match(/^\/api\/hooks\/([^/]+)(?:\/([^/]+))?$/);
+    if (hooksMatch) {
+        return {
+            path: `/api/hooks/:type${hooksMatch[2] ? '/:name' : ''}`,
+            params: {
+                type: decodeURIComponent(hooksMatch[1]),
+                ...(hooksMatch[2] ? { name: decodeURIComponent(hooksMatch[2]) } : {}),
+            },
+            query,
+        };
+    }
     return { path: pathPart, params: {}, query };
 }
 function readBody(req) {
@@ -407,9 +496,8 @@ function jsonResponse(res, status, data) {
     res.end(JSON.stringify(data));
 }
 // ─── Route handlers ──────────────────────────────────────────────────────────
-// Default threads directory — can be overridden via basePath in server options
 let THREADS_DIR = path.join(os.homedir(), '.cumulus', 'threads');
-/** Override the threads directory (used by startGatewayServer when basePath is set) */
+/** Override the threads directory (used by tests to isolate from production) */
 export function setThreadsDir(dir) {
     THREADS_DIR = dir;
 }
@@ -773,6 +861,8 @@ async function handleUpdateThreadConfig(threadName, req, res) {
             updates.devices = body.devices;
         if (body.gitRemoteUrl !== undefined)
             updates.gitRemoteUrl = body.gitRemoteUrl;
+        if (body.model !== undefined)
+            updates.model = body.model;
         await saveThreadConfig(threadName, updates);
         const updated = await loadThreadConfig(threadName);
         jsonResponse(res, 200, { config: updated });
@@ -781,6 +871,200 @@ async function handleUpdateThreadConfig(threadName, req, res) {
         jsonResponse(res, 500, { error: String(err) });
     }
 }
+// ─── Pipeline dashboard handlers ───────────────────────────
+const STAGE_ORDER = [
+    'prospect',
+    'outreach',
+    'warm-engagement',
+    'proposal',
+    'onboarding',
+    'active-client',
+    'archived',
+];
+async function handleGetDashboard(res) {
+    try {
+        const files = fs.readdirSync(THREADS_DIR);
+        const threads = [];
+        for (const file of files) {
+            if (!file.endsWith('.config.json'))
+                continue;
+            const threadName = file.replace('.config.json', '');
+            const config = await loadThreadConfig(threadName);
+            // Skip threads without pipeline metadata
+            if (!config.pipeline?.stage)
+                continue;
+            // Get last activity from JSONL last line timestamp (read last 1KB only)
+            const historyPath = path.join(THREADS_DIR, `${threadName}.jsonl`);
+            let lastActivity;
+            try {
+                const stats = fs.statSync(historyPath);
+                if (stats.size > 0) {
+                    const fd = fs.openSync(historyPath, 'r');
+                    const bufferSize = Math.min(1024, stats.size);
+                    const buffer = Buffer.alloc(bufferSize);
+                    fs.readSync(fd, buffer, 0, bufferSize, stats.size - bufferSize);
+                    fs.closeSync(fd);
+                    const lines = buffer.toString().trim().split('\n');
+                    const lastLine = lines[lines.length - 1];
+                    if (lastLine) {
+                        try {
+                            const msg = JSON.parse(lastLine);
+                            lastActivity = msg.timestamp;
+                        }
+                        catch {
+                            // Malformed last line — skip
+                        }
+                    }
+                }
+            }
+            catch {
+                // History file may not exist
+            }
+            threads.push({
+                name: threadName,
+                stage: config.pipeline.stage,
+                score: config.pipeline.score ?? 0,
+                lastContact: config.pipeline.lastContact,
+                lastActivity,
+                nextAction: config.pipeline.nextAction,
+                nextActionDate: config.pipeline.nextActionDate,
+            });
+        }
+        // Group by stage
+        const byStage = {};
+        for (const stage of STAGE_ORDER) {
+            byStage[stage] = threads.filter(t => t.stage === stage);
+        }
+        // Sort threads within each stage by score (descending)
+        for (const stage of Object.keys(byStage)) {
+            byStage[stage].sort((a, b) => b.score - a.score);
+        }
+        jsonResponse(res, 200, {
+            threads: byStage,
+            stats: {
+                total: threads.length,
+                byStage: Object.fromEntries(STAGE_ORDER.map(s => [s, byStage[s].length])),
+            },
+        });
+    }
+    catch (err) {
+        jsonResponse(res, 500, { error: String(err) });
+    }
+}
+async function handleUpdatePipeline(threadName, req, res) {
+    let body;
+    try {
+        const raw = await readBody(req);
+        body = JSON.parse(raw);
+    }
+    catch {
+        jsonResponse(res, 400, { error: 'Invalid JSON body' });
+        return;
+    }
+    // Validate stage if provided
+    if (body.stage !== undefined &&
+        !STAGE_ORDER.includes(body.stage)) {
+        jsonResponse(res, 400, { error: `Invalid stage. Must be one of: ${STAGE_ORDER.join(', ')}` });
+        return;
+    }
+    // Validate score if provided
+    if (body.score !== undefined) {
+        const score = Number(body.score);
+        if (isNaN(score) || score < 0 || score > 100) {
+            jsonResponse(res, 400, { error: 'Score must be a number between 0 and 100' });
+            return;
+        }
+    }
+    try {
+        // Read-merge-write to avoid clobbering sibling pipeline fields
+        const existing = await loadThreadConfig(threadName);
+        const pipelineUpdates = {};
+        for (const key of ['stage', 'score', 'lastContact', 'nextAction', 'nextActionDate']) {
+            if (body[key] !== undefined)
+                pipelineUpdates[key] = body[key];
+        }
+        existing.pipeline = { ...existing.pipeline, ...pipelineUpdates };
+        await saveThreadConfig(threadName, existing);
+        jsonResponse(res, 200, { pipeline: existing.pipeline, thread: threadName });
+    }
+    catch (err) {
+        jsonResponse(res, 500, { error: String(err) });
+    }
+}
+// ─── Schedule handlers ─────────────────────────────────────
+async function handleListSchedules(threadName, res) {
+    try {
+        const config = await loadThreadConfig(threadName);
+        jsonResponse(res, 200, { schedules: config.schedules || [] });
+    }
+    catch (err) {
+        jsonResponse(res, 500, { error: String(err) });
+    }
+}
+async function handleCreateSchedule(threadName, req, res, scheduler) {
+    let body;
+    try {
+        const raw = await readBody(req);
+        body = JSON.parse(raw);
+    }
+    catch {
+        jsonResponse(res, 400, { error: 'Invalid JSON body' });
+        return;
+    }
+    if (!body.id || !body.trigger || !body.message) {
+        jsonResponse(res, 400, { error: 'Missing required fields: id, trigger, message' });
+        return;
+    }
+    if (body.trigger !== 'once' && body.trigger !== 'cron') {
+        jsonResponse(res, 400, { error: 'trigger must be "once" or "cron"' });
+        return;
+    }
+    if (body.trigger === 'once' && !body.at) {
+        jsonResponse(res, 400, { error: 'Once trigger requires "at" datetime field' });
+        return;
+    }
+    if (body.trigger === 'cron' && !body.cron) {
+        jsonResponse(res, 400, { error: 'Cron trigger requires "cron" expression field' });
+        return;
+    }
+    try {
+        const config = await loadThreadConfig(threadName);
+        const schedules = config.schedules || [];
+        // Replace if same ID exists
+        const filtered = schedules.filter(s => s.id !== body.id);
+        filtered.push({
+            id: body.id,
+            trigger: body.trigger,
+            ...(body.at ? { at: body.at } : {}),
+            ...(body.cron ? { cron: body.cron } : {}),
+            message: body.message,
+        });
+        await saveThreadConfig(threadName, { schedules: filtered });
+        scheduler?.reloadThread(threadName);
+        jsonResponse(res, 200, { created: true, id: body.id, trigger: body.trigger });
+    }
+    catch (err) {
+        jsonResponse(res, 500, { error: String(err) });
+    }
+}
+async function handleDeleteSchedule(threadName, scheduleId, res, scheduler) {
+    try {
+        const config = await loadThreadConfig(threadName);
+        const schedules = config.schedules || [];
+        const before = schedules.length;
+        const filtered = schedules.filter(s => s.id !== scheduleId);
+        if (filtered.length === before) {
+            jsonResponse(res, 404, { error: `Schedule "${scheduleId}" not found` });
+            return;
+        }
+        await saveThreadConfig(threadName, { schedules: filtered });
+        scheduler?.reloadThread(threadName);
+        jsonResponse(res, 200, { cancelled: true, id: scheduleId });
+    }
+    catch (err) {
+        jsonResponse(res, 500, { error: String(err) });
+    }
+}
 async function handleListTemplates(res) {
     try {
         const templates = listTemplates();
@@ -1002,16 +1286,95 @@ async function handleNotify(req, res) {
         jsonResponse(res, 500, { error: String(err) });
     }
 }
+// ─── Media upload ────────────────────────────────────────────────────────────
+async function handleMediaUpload(req, res) {
+    const mediaDir = getMediaDir();
+    // Read raw body (binary)
+    const chunks = [];
+    for await (const chunk of req) {
+        chunks.push(chunk);
+    }
+    const body = Buffer.concat(chunks);
+    if (body.length === 0) {
+        jsonResponse(res, 400, { error: 'Empty request body' });
+        return;
+    }
+    // Max 50MB
+    if (body.length > 50 * 1024 * 1024) {
+        jsonResponse(res, 413, { error: 'File too large (max 50MB)' });
+        return;
+    }
+    const contentType = req.headers['content-type'] || '';
+    if (contentType.includes('multipart/form-data')) {
+        const boundaryMatch = contentType.match(/boundary=(.+)/);
+        if (!boundaryMatch) {
+            jsonResponse(res, 400, { error: 'Missing multipart boundary' });
+            return;
+        }
+        const boundary = boundaryMatch[1];
+        const parts = parseMultipart(body, boundary);
+        if (parts.length === 0) {
+            jsonResponse(res, 400, { error: 'No file found in multipart body' });
+            return;
+        }
+        const results = [];
+        for (const part of parts) {
+            const result = saveMediaFile(mediaDir, part.filename || 'upload', part.data);
+            results.push(result);
+        }
+        jsonResponse(res, 200, { files: results });
+    }
+    else {
+        // Raw binary upload — filename from query param or X-Filename header
+        const urlObj = new URL(req.url || '/', `http://${req.headers.host}`);
+        const filename = urlObj.searchParams.get('filename') || req.headers['x-filename'] || 'upload';
+        const result = saveMediaFile(mediaDir, filename, body);
+        jsonResponse(res, 200, result);
+    }
+}
+function parseMultipart(body, boundary) {
+    const parts = [];
+    const bodyStr = body.toString('binary');
+    const sections = bodyStr.split(`--${boundary}`);
+    for (const section of sections) {
+        if (section === '' || section.startsWith('--'))
+            continue;
+        const headerEnd = section.indexOf('\r\n\r\n');
+        if (headerEnd === -1)
+            continue;
+        const headers = section.slice(0, headerEnd);
+        const dataStr = section.slice(headerEnd + 4);
+        const trimmed = dataStr.endsWith('\r\n') ? dataStr.slice(0, -2) : dataStr;
+        const filenameMatch = headers.match(/filename="([^"]+)"/);
+        const ctMatch = headers.match(/Content-Type:\s*(.+)/i);
+        parts.push({
+            filename: filenameMatch?.[1],
+            contentType: ctMatch?.[1]?.trim(),
+            data: Buffer.from(trimmed, 'binary'),
+        });
+    }
+    return parts;
+}
+function saveMediaFile(mediaDir, originalFilename, data) {
+    const ext = path.extname(originalFilename).toLowerCase() || '.bin';
+    const baseName = path
+        .basename(originalFilename, ext)
+        .replace(/[^a-zA-Z0-9_-]/g, '_')
+        .slice(0, 64);
+    const hash = crypto.createHash('md5').update(data).digest('hex').slice(0, 8);
+    const filename = `${baseName}-${hash}${ext}`;
+    fs.writeFileSync(path.join(mediaDir, filename), data);
+    return {
+        filename,
+        url: `/media/${filename}`,
+        size: data.length,
+    };
+}
 // ─── Server ──────────────────────────────────────────────────────────────────
 export async function startGatewayServer(options) {
     const { port, ...pipelineOpts } = options;
     let currentApiKeys = options.apiKeys;
-    // Override threads directory if basePath is provided
-    if (options.basePath) {
-        const threadsDir = path.join(options.basePath, 'threads');
-        fs.mkdirSync(threadsDir, { recursive: true });
-        setThreadsDir(threadsDir);
-    }
+    let scheduler;
     // Create or use provided executor registry
     const executorRegistry = options.executorRegistry || new ExecutorRegistry();
     pipelineOpts.executorRegistry = executorRegistry;
@@ -1036,7 +1399,11 @@ export async function startGatewayServer(options) {
         const { path: routePath, params, query } = parseRoute(req.url || '/');
         // Health check — no auth required
         if (routePath === '/health' && req.method === 'GET') {
-            jsonResponse(res, 200, { status: 'ok', timestamp: new Date().toISOString() });
+            jsonResponse(res, 200, {
+                status: 'ok',
+                timestamp: new Date().toISOString(),
+                activeSubprocesses: getActiveSubprocessCount(),
+            });
             return;
         }
         // Static files — no auth required (/chat, /widget.js)
@@ -1048,6 +1415,12 @@ export async function startGatewayServer(options) {
             handleGetVapidKey(res, vapidConfig);
             return;
         }
+        // POST /api/hooks/:type — inbound webhooks (no API key auth — uses signature verification)
+        if ((routePath === '/api/hooks/:type' || routePath === '/api/hooks/:type/:name') &&
+            req.method === 'POST') {
+            await handleWebhook(params.type, params.name, req, res, pipelineOpts, options.hooks);
+            return;
+        }
         // All other routes require auth
         if (!authenticate(req, res, currentApiKeys))
             return;
@@ -1098,6 +1471,34 @@ export async function startGatewayServer(options) {
                 await handleUpdateThreadConfig(params.name, req, res);
                 return;
             }
+            // PUT /api/thread/:name/pipeline — update pipeline metadata (for dashboard drag-drop)
+            if (routePath === '/api/thread/:name/pipeline' && req.method === 'PUT') {
+                await handleUpdatePipeline(params.name, req, res);
+                return;
+            }
+            // GET /api/thread/:name/schedules — list schedules
+            if (routePath === '/api/thread/:name/schedules' && req.method === 'GET') {
+                await handleListSchedules(params.name, res);
+                return;
+            }
+            // POST /api/thread/:name/schedules — create/update a schedule
+            if (routePath === '/api/thread/:name/schedules' && req.method === 'POST') {
+                await handleCreateSchedule(params.name, req, res, scheduler);
+                return;
+            }
+            // DELETE /api/thread/:name/schedules/:id — cancel a schedule
+            if (routePath.startsWith('/api/thread/:name/schedules/') && req.method === 'DELETE') {
+                const scheduleId = routePath.replace('/api/thread/:name/schedules/', '');
+                if (scheduleId) {
+                    await handleDeleteSchedule(params.name, decodeURIComponent(scheduleId), res, scheduler);
+                    return;
+                }
+            }
+            // GET /api/dashboard — pipeline dashboard data
+            if (routePath === '/api/dashboard' && req.method === 'GET') {
+                await handleGetDashboard(res);
+                return;
+            }
             // GET /api/threads
             if (routePath === '/api/threads' && req.method === 'GET') {
                 await handleListThreads(res, executorRegistry);
@@ -1113,6 +1514,26 @@ export async function startGatewayServer(options) {
                 await handleListTemplates(res);
                 return;
             }
+            // GET /api/models — list available models for per-thread selection
+            if (routePath === '/api/models' && req.method === 'GET') {
+                // Build models list with global default marked
+                let models = (options.models || []).map(m => ({ ...m }));
+                if (models.length === 0) {
+                    // Fallback: just Claude
+                    models = [
+                        { id: 'claude', label: 'Claude (CLI)', provider: 'claude-cli', default: true },
+                    ];
+                }
+                else {
+                    // Clear all defaults, then set the global default (or first if none specified)
+                    const globalDefault = options.model || models[0].id;
+                    for (const m of models) {
+                        m.default = m.id === globalDefault ? true : undefined;
+                    }
+                }
+                jsonResponse(res, 200, { models });
+                return;
+            }
             // GET /api/agents — list agents
             if (routePath === '/api/agents' && req.method === 'GET') {
                 await handleListAgents(res);
@@ -1148,6 +1569,11 @@ export async function startGatewayServer(options) {
                 await handleNotify(req, res);
                 return;
             }
+            // POST /api/media/upload — upload files to media directory
+            if (routePath === '/api/media/upload' && req.method === 'POST') {
+                await handleMediaUpload(req, res);
+                return;
+            }
             // 404
             jsonResponse(res, 404, { error: 'Not found' });
         }
@@ -1210,6 +1636,9 @@ export async function startGatewayServer(options) {
                 setBroadcastToThread: (fn) => {
                     pipelineOpts.broadcastToThread = fn;
                 },
+                setScheduler: (s) => {
+                    scheduler = s;
+                },
                 close: async () => {
                     // Abort any in-flight message processing
                     threadQueues.clear();