create-ironclaws 1.0.0

package/template/src/group-queue.ts

@@ -0,0 +1,363 @@
+import { ChildProcess } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+
+import { BASE_RETRY_MS, DATA_DIR, MAX_CONCURRENT_CONTAINERS, MAX_RETRIES } from './config.js';
+import { logger } from './logger.js';
+
+interface QueuedTask {
+  id: string;
+  groupJid: string;
+  fn: () => Promise<void>;
+}
+
+
+interface GroupState {
+  active: boolean;
+  idleWaiting: boolean;
+  isTaskContainer: boolean;
+  runningTaskId: string | null;
+  pendingMessages: boolean;
+  pendingTasks: QueuedTask[];
+  process: ChildProcess | null;
+  containerName: string | null;
+  groupFolder: string | null;
+  retryCount: number;
+}
+
+export class GroupQueue {
+  private groups = new Map<string, GroupState>();
+  private activeCount = 0;
+  private waitingGroups: string[] = [];
+  private processMessagesFn: ((groupJid: string) => Promise<boolean>) | null =
+    null;
+  private shuttingDown = false;
+
+  private getGroup(groupJid: string): GroupState {
+    let state = this.groups.get(groupJid);
+    if (!state) {
+      state = {
+        active: false,
+        idleWaiting: false,
+        isTaskContainer: false,
+        runningTaskId: null,
+        pendingMessages: false,
+        pendingTasks: [],
+        process: null,
+        containerName: null,
+        groupFolder: null,
+        retryCount: 0,
+      };
+      this.groups.set(groupJid, state);
+    }
+    return state;
+  }
+
+  setProcessMessagesFn(fn: (groupJid: string) => Promise<boolean>): void {
+    this.processMessagesFn = fn;
+  }
+
+  enqueueMessageCheck(groupJid: string): void {
+    if (this.shuttingDown) return;
+
+    const state = this.getGroup(groupJid);
+
+    if (state.active) {
+      state.pendingMessages = true;
+      logger.debug({ groupJid }, 'Container active, message queued');
+      return;
+    }
+
+    if (this.activeCount >= MAX_CONCURRENT_CONTAINERS) {
+      state.pendingMessages = true;
+      if (!this.waitingGroups.includes(groupJid)) {
+        this.waitingGroups.push(groupJid);
+      }
+      logger.debug(
+        { groupJid, activeCount: this.activeCount },
+        'At concurrency limit, message queued',
+      );
+      return;
+    }
+
+    this.runForGroup(groupJid, 'messages').catch((err) =>
+      logger.error({ groupJid, err }, 'Unhandled error in runForGroup'),
+    );
+  }
+
+  enqueueTask(groupJid: string, taskId: string, fn: () => Promise<void>): void {
+    if (this.shuttingDown) return;
+
+    const state = this.getGroup(groupJid);
+
+    // Prevent double-queuing: check both pending and currently-running task
+    if (state.runningTaskId === taskId) {
+      logger.debug({ groupJid, taskId }, 'Task already running, skipping');
+      return;
+    }
+    if (state.pendingTasks.some((t) => t.id === taskId)) {
+      logger.debug({ groupJid, taskId }, 'Task already queued, skipping');
+      return;
+    }
+
+    if (state.active) {
+      state.pendingTasks.push({ id: taskId, groupJid, fn });
+      if (state.idleWaiting) {
+        this.closeStdin(groupJid);
+      }
+      logger.debug({ groupJid, taskId }, 'Container active, task queued');
+      return;
+    }
+
+    if (this.activeCount >= MAX_CONCURRENT_CONTAINERS) {
+      state.pendingTasks.push({ id: taskId, groupJid, fn });
+      if (!this.waitingGroups.includes(groupJid)) {
+        this.waitingGroups.push(groupJid);
+      }
+      logger.debug(
+        { groupJid, taskId, activeCount: this.activeCount },
+        'At concurrency limit, task queued',
+      );
+      return;
+    }
+
+    // Run immediately
+    this.runTask(groupJid, { id: taskId, groupJid, fn }).catch((err) =>
+      logger.error({ groupJid, taskId, err }, 'Unhandled error in runTask'),
+    );
+  }
+
+  registerProcess(
+    groupJid: string,
+    proc: ChildProcess,
+    containerName: string,
+    groupFolder?: string,
+  ): void {
+    const state = this.getGroup(groupJid);
+    state.process = proc;
+    state.containerName = containerName;
+    if (groupFolder) state.groupFolder = groupFolder;
+  }
+
+  /**
+   * Mark the container as idle-waiting (finished work, waiting for IPC input).
+   * If tasks are pending, preempt the idle container immediately.
+   */
+  notifyIdle(groupJid: string): void {
+    const state = this.getGroup(groupJid);
+    state.idleWaiting = true;
+    if (state.pendingTasks.length > 0) {
+      this.closeStdin(groupJid);
+    }
+  }
+
+  /**
+   * Send a follow-up message to the active container via IPC file.
+   * Returns true if the message was written, false if no active container.
+   */
+  sendMessage(groupJid: string, text: string): boolean {
+    const state = this.getGroup(groupJid);
+    if (!state.active || !state.groupFolder || state.isTaskContainer)
+      return false;
+    state.idleWaiting = false; // Agent is about to receive work, no longer idle
+
+    const inputDir = path.join(DATA_DIR, 'ipc', state.groupFolder, 'input');
+    try {
+      fs.mkdirSync(inputDir, { recursive: true });
+      const filename = `${Date.now()}-${Math.random().toString(36).slice(2, 6)}.json`;
+      const filepath = path.join(inputDir, filename);
+      const tempPath = `${filepath}.tmp`;
+      fs.writeFileSync(tempPath, JSON.stringify({ type: 'message', text }));
+      fs.renameSync(tempPath, filepath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Signal the active container to wind down by writing a close sentinel.
+   */
+  closeStdin(groupJid: string): void {
+    const state = this.getGroup(groupJid);
+    if (!state.active || !state.groupFolder) return;
+
+    const inputDir = path.join(DATA_DIR, 'ipc', state.groupFolder, 'input');
+    try {
+      fs.mkdirSync(inputDir, { recursive: true });
+      fs.writeFileSync(path.join(inputDir, '_close'), '');
+    } catch {
+      // ignore
+    }
+  }
+
+  private async runForGroup(
+    groupJid: string,
+    reason: 'messages' | 'drain',
+  ): Promise<void> {
+    const state = this.getGroup(groupJid);
+    state.active = true;
+    state.idleWaiting = false;
+    state.isTaskContainer = false;
+    state.pendingMessages = false;
+    this.activeCount++;
+
+    logger.debug(
+      { groupJid, reason, activeCount: this.activeCount },
+      'Starting container for group',
+    );
+
+    try {
+      if (this.processMessagesFn) {
+        const success = await this.processMessagesFn(groupJid);
+        if (success) {
+          state.retryCount = 0;
+        } else {
+          this.scheduleRetry(groupJid, state);
+        }
+      }
+    } catch (err) {
+      logger.error({ groupJid, err }, 'Error processing messages for group');
+      this.scheduleRetry(groupJid, state);
+    } finally {
+      state.active = false;
+      state.process = null;
+      state.containerName = null;
+      state.groupFolder = null;
+      this.activeCount--;
+      this.drainGroup(groupJid);
+    }
+  }
+
+  private async runTask(groupJid: string, task: QueuedTask): Promise<void> {
+    const state = this.getGroup(groupJid);
+    state.active = true;
+    state.idleWaiting = false;
+    state.isTaskContainer = true;
+    state.runningTaskId = task.id;
+    this.activeCount++;
+
+    logger.debug(
+      { groupJid, taskId: task.id, activeCount: this.activeCount },
+      'Running queued task',
+    );
+
+    try {
+      await task.fn();
+    } catch (err) {
+      logger.error({ groupJid, taskId: task.id, err }, 'Error running task');
+    } finally {
+      state.active = false;
+      state.isTaskContainer = false;
+      state.runningTaskId = null;
+      state.process = null;
+      state.containerName = null;
+      state.groupFolder = null;
+      this.activeCount--;
+      this.drainGroup(groupJid);
+    }
+  }
+
+  private scheduleRetry(groupJid: string, state: GroupState): void {
+    state.retryCount++;
+    if (state.retryCount > MAX_RETRIES) {
+      logger.error(
+        { groupJid, retryCount: state.retryCount },
+        'Max retries exceeded, dropping messages (will retry on next incoming message)',
+      );
+      state.retryCount = 0;
+      return;
+    }
+
+    const delayMs = BASE_RETRY_MS * Math.pow(2, state.retryCount - 1);
+    logger.info(
+      { groupJid, retryCount: state.retryCount, delayMs },
+      'Scheduling retry with backoff',
+    );
+    setTimeout(() => {
+      if (!this.shuttingDown) {
+        this.enqueueMessageCheck(groupJid);
+      }
+    }, delayMs);
+  }
+
+  private drainGroup(groupJid: string): void {
+    if (this.shuttingDown) return;
+
+    const state = this.getGroup(groupJid);
+
+    // Tasks first (they won't be re-discovered from SQLite like messages)
+    if (state.pendingTasks.length > 0) {
+      const task = state.pendingTasks.shift()!;
+      this.runTask(groupJid, task).catch((err) =>
+        logger.error(
+          { groupJid, taskId: task.id, err },
+          'Unhandled error in runTask (drain)',
+        ),
+      );
+      return;
+    }
+
+    // Then pending messages
+    if (state.pendingMessages) {
+      this.runForGroup(groupJid, 'drain').catch((err) =>
+        logger.error(
+          { groupJid, err },
+          'Unhandled error in runForGroup (drain)',
+        ),
+      );
+      return;
+    }
+
+    // Nothing pending for this group; check if other groups are waiting for a slot
+    this.drainWaiting();
+  }
+
+  private drainWaiting(): void {
+    while (
+      this.waitingGroups.length > 0 &&
+      this.activeCount < MAX_CONCURRENT_CONTAINERS
+    ) {
+      const nextJid = this.waitingGroups.shift()!;
+      const state = this.getGroup(nextJid);
+
+      // Prioritize tasks over messages
+      if (state.pendingTasks.length > 0) {
+        const task = state.pendingTasks.shift()!;
+        this.runTask(nextJid, task).catch((err) =>
+          logger.error(
+            { groupJid: nextJid, taskId: task.id, err },
+            'Unhandled error in runTask (waiting)',
+          ),
+        );
+      } else if (state.pendingMessages) {
+        this.runForGroup(nextJid, 'drain').catch((err) =>
+          logger.error(
+            { groupJid: nextJid, err },
+            'Unhandled error in runForGroup (waiting)',
+          ),
+        );
+      }
+      // If neither pending, skip this group
+    }
+  }
+
+  async shutdown(_gracePeriodMs: number): Promise<void> {
+    this.shuttingDown = true;
+
+    // Count active containers but don't kill them — they'll finish on their own
+    // via idle timeout or container timeout. The --rm flag cleans them up on exit.
+    // This prevents WhatsApp reconnection restarts from killing working agents.
+    const activeContainers: string[] = [];
+    for (const [_jid, state] of this.groups) {
+      if (state.process && !state.process.killed && state.containerName) {
+        activeContainers.push(state.containerName);
+      }
+    }
+
+    logger.info(
+      { activeCount: this.activeCount, detachedContainers: activeContainers },
+      'GroupQueue shutting down (containers detached, not killed)',
+    );
+  }
+}

package/template/src/http-server.ts

@@ -0,0 +1,343 @@
+/**
+ * HTTP Server for NanoClaw
+ *
+ * Receives questionnaire questions via POST and routes them
+ * to the questionnaire-assistant OpenShell sandbox agent.
+ * Runs alongside the existing Slack connection.
+ */
+import http from 'http';
+
+import { findExactMatch, saveQuestionnaireMetric } from './db.js';
+import { readEnvFile } from './env.js';
+import { logger } from './logger.js';
+import { runContainerAgent, ContainerOutput } from './container-runner.js';
+import type { RegisteredGroup } from './types.js';
+
+// ─── Concurrency limiter for sandbox execs ───────────────────────────────────
+const MAX_CONCURRENT_EXECS = 10;
+let activeExecs = 0;
+const execQueue: Array<() => void> = [];
+
+function acquireExecSlot(): Promise<void> {
+  if (activeExecs < MAX_CONCURRENT_EXECS) {
+    activeExecs++;
+    return Promise.resolve();
+  }
+  return new Promise((resolve) => {
+    execQueue.push(() => {
+      activeExecs++;
+      resolve();
+    });
+  });
+}
+
+function releaseExecSlot(): void {
+  activeExecs--;
+  const next = execQueue.shift();
+  if (next) next();
+}
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+interface QuestionnaireRequest {
+  question: string;
+  systemPrompt: string | null;
+  userEmail: string;
+  documentUrl: string;
+  documentId: string;
+  documentName: string;
+}
+
+interface Citation {
+  source: string;
+  title: string;
+  url: string;
+  answer_source: string;
+}
+
+interface QuestionnaireResponse {
+  answer: string;
+  citations: Citation[];
+  answer_source: string;
+  confidence: string;
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function jsonResponse(
+  res: http.ServerResponse,
+  statusCode: number,
+  body: unknown,
+): void {
+  const json = JSON.stringify(body);
+  res.writeHead(statusCode, {
+    'Content-Type': 'application/json',
+    'Access-Control-Allow-Origin': '*',
+    'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+    'Access-Control-Allow-Headers': 'Content-Type, Authorization',
+  });
+  res.end(json);
+}
+
+function readBody(req: http.IncomingMessage): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let size = 0;
+    const MAX_BODY = 1_048_576; // 1 MB
+
+    req.on('data', (chunk: Buffer) => {
+      size += chunk.length;
+      if (size > MAX_BODY) {
+        req.destroy();
+        reject(new Error('Request body too large'));
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    req.on('error', reject);
+  });
+}
+
+// ─── Route handlers ─────────────────────────────────────────────────────────
+
+function handleStatus(
+  _req: http.IncomingMessage,
+  res: http.ServerResponse,
+): void {
+  jsonResponse(res, 200, { status: 'ok', uptime: process.uptime() });
+}
+
+async function handleQuestionnaireAnswer(
+  req: http.IncomingMessage,
+  res: http.ServerResponse,
+  authToken: string,
+  registeredGroups: () => Record<string, RegisteredGroup>,
+): Promise<void> {
+  // ── Auth ────────────────────────────────────────────────────────────────
+  const authHeader = req.headers.authorization;
+  if (!authHeader || authHeader !== `Bearer ${authToken}`) {
+    jsonResponse(res, 401, { error: 'Unauthorized' });
+    return;
+  }
+
+  // ── Parse body ─────────────────────────────────────────────────────────
+  let body: QuestionnaireRequest;
+  try {
+    const raw = await readBody(req);
+    body = JSON.parse(raw) as QuestionnaireRequest;
+  } catch (err) {
+    jsonResponse(res, 400, {
+      error: 'Invalid JSON body',
+      detail: err instanceof Error ? err.message : String(err),
+    });
+    return;
+  }
+
+  const { question, systemPrompt, userEmail, documentUrl, documentId, documentName } =
+    body;
+
+  if (!question || typeof question !== 'string') {
+    jsonResponse(res, 400, { error: 'Missing required field: question' });
+    return;
+  }
+
+  const startTime = Date.now();
+
+  // ── Exact-match cache ──────────────────────────────────────────────────
+  const cached = findExactMatch(question);
+  if (cached) {
+    logger.info({ question: question.slice(0, 80) }, 'Questionnaire cache hit');
+    jsonResponse(res, 200, cached);
+    return;
+  }
+
+  // ── Find the questionnaire-assistant group ─────────────────────────────
+  const groups = registeredGroups();
+  let group: RegisteredGroup | undefined;
+  for (const g of Object.values(groups)) {
+    if (g.folder === 'questionnaire-assistant') {
+      group = g;
+      break;
+    }
+  }
+
+  if (!group) {
+    jsonResponse(res, 503, {
+      error: 'questionnaire-assistant agent is not registered',
+    });
+    return;
+  }
+
+  // ── Run the agent (with concurrency limit) ─────────────────────────────
+  const prompt = `<question>${question}</question>\n<system_prompt>${systemPrompt || ''}</system_prompt>\n<user_email>${userEmail}</user_email>`;
+
+  // Wait for an exec slot
+  await acquireExecSlot();
+  logger.info(
+    { question: question.slice(0, 60), activeExecs, queued: execQueue.length },
+    'Exec slot acquired',
+  );
+
+  // Use a Promise that resolves when onOutput fires — don't wait for container exit.
+  // Same pattern as Slack: respond immediately when the agent produces output.
+  try {
+    const resultText = await new Promise<string>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        reject(new Error('Agent timed out after 5 minutes'));
+      }, 300_000);
+
+      runContainerAgent(
+        group,
+        {
+          prompt,
+          groupFolder: 'questionnaire-assistant',
+          chatJid: 'http:questionnaire',
+          isMain: false,
+          assistantName: 'Questionnaire Assistant',
+        },
+        () => {}, // onProcess — not needed for HTTP
+        async (streamedOutput) => {
+          // First non-empty result resolves the promise immediately
+          if (streamedOutput.result) {
+            clearTimeout(timeout);
+            resolve(streamedOutput.result);
+          }
+        },
+      ).catch((err) => {
+        clearTimeout(timeout);
+        reject(err);
+      });
+    });
+
+    // ── Parse agent response ───────────────────────────────────────────────
+    let parsed: QuestionnaireResponse;
+    try {
+      parsed = JSON.parse(resultText.trim());
+    } catch {
+      try {
+        const jsonMatch = resultText.match(/\{[\s\S]*"answer"\s*:\s*"[\s\S]*\}/);
+        if (jsonMatch) {
+          parsed = JSON.parse(jsonMatch[0]);
+        } else {
+          throw new Error('No JSON found');
+        }
+      } catch {
+        parsed = {
+          answer: resultText,
+          citations: [],
+          answer_source: 'ai',
+          confidence: 'low',
+        };
+      }
+    }
+
+    const responseTimeMs = Date.now() - startTime;
+
+    saveQuestionnaireMetric({
+      user_email: userEmail || null,
+      document_url: documentUrl || null,
+      document_id: documentId || null,
+      question,
+      answer: parsed.answer,
+      response_time_ms: responseTimeMs,
+      was_answered: parsed.answer ? 1 : 0,
+      answer_reference: parsed.answer_source || null,
+      category: ((parsed as unknown as Record<string, unknown>).category as string) || null,
+      confidence: parsed.confidence || null,
+    });
+
+    logger.info(
+      {
+        question: question.slice(0, 80),
+        responseTimeMs,
+        confidence: parsed.confidence,
+      },
+      'Questionnaire answered',
+    );
+
+    jsonResponse(res, 200, {
+      answer: parsed.answer,
+      citations: parsed.citations || [],
+      answer_source: parsed.answer_source || 'ai',
+      confidence: parsed.confidence || 'low',
+    });
+  } catch (err) {
+    const responseTimeMs = Date.now() - startTime;
+    logger.error(
+      { err, question: question.slice(0, 80), responseTimeMs },
+      'Questionnaire request failed',
+    );
+    jsonResponse(res, 500, {
+      error: 'Internal server error',
+      detail: err instanceof Error ? err.message : String(err),
+    });
+  } finally {
+    releaseExecSlot();
+  }
+}
+
+// ─── Server ─────────────────────────────────────────────────────────────────
+
+export function startHttpServer(opts: {
+  registeredGroups: () => Record<string, RegisteredGroup>;
+}): void {
+  const envConfig = readEnvFile(['API_AUTH_TOKEN', 'HTTP_PORT']);
+  const authToken = envConfig.API_AUTH_TOKEN;
+  const port = parseInt(envConfig.HTTP_PORT || '3000', 10);
+
+  if (!authToken) {
+    logger.warn(
+      'API_AUTH_TOKEN not set — HTTP server will reject all authenticated requests',
+    );
+  }
+
+  const server = http.createServer((req, res) => {
+    const url = new URL(req.url || '/', `http://localhost:${port}`);
+    const method = req.method?.toUpperCase() || 'GET';
+
+    // CORS preflight
+    if (method === 'OPTIONS') {
+      res.writeHead(204, {
+        'Access-Control-Allow-Origin': '*',
+        'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+        'Access-Control-Allow-Headers': 'Content-Type, Authorization',
+        'Access-Control-Max-Age': '86400',
+      });
+      res.end();
+      return;
+    }
+
+    // Route dispatch
+    if (url.pathname === '/status' && method === 'GET') {
+      handleStatus(req, res);
+      return;
+    }
+
+    if (url.pathname === '/api/questionnaire/answer' && method === 'POST') {
+      handleQuestionnaireAnswer(
+        req,
+        res,
+        authToken || '',
+        opts.registeredGroups,
+      ).catch((err) => {
+        logger.error({ err }, 'Unhandled error in questionnaire handler');
+        if (!res.headersSent) {
+          jsonResponse(res, 500, { error: 'Internal server error' });
+        }
+      });
+      return;
+    }
+
+    // 404 for everything else
+    jsonResponse(res, 404, { error: 'Not found' });
+  });
+
+  server.listen(port, () => {
+    logger.info({ port }, 'HTTP server listening');
+  });
+
+  server.on('error', (err) => {
+    logger.error({ err, port }, 'HTTP server error');
+  });
+}