@mooncompany/uplink-chat 0.5.0

package/server/routes/files.js

@@ -0,0 +1,307 @@
+/**
+ * File Routes - General file upload with text extraction
+ * Supports: PDF, DOCX, XLSX/CSV, plain text, code files
+ * Extracted text is saved alongside originals for agent access via `read` tool
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { randomUUID } from 'crypto';
+import { fileTypeFromBuffer } from 'file-type';
+import { badRequest, internalError, ErrorCodes } from '../../utils/errors.js';
+
+// Lazy-load heavy parsers (only when needed)
+let pdfParse = null;
+let mammoth = null;
+let ExcelJS = null;
+
+async function loadPdfParse() {
+  if (!pdfParse) {
+    const mod = await import('pdf-parse');
+    pdfParse = mod.PDFParse || mod.default || mod;
+  }
+  return pdfParse;
+}
+
+async function loadMammoth() {
+  if (!mammoth) {
+    const mod = await import('mammoth');
+    mammoth = mod.default || mod;
+  }
+  return mammoth;
+}
+
+async function loadExcelJS() {
+  if (!ExcelJS) {
+    const mod = await import('exceljs');
+    ExcelJS = mod.default || mod;
+  }
+  return ExcelJS;
+}
+
+import { MAX_FILE_EXTRACT_SIZE, FILE_EXTRACTION_TIMEOUT_MS, FILE_CLEANUP_DELAY_MS } from '../config.js';
+
+// Max text extraction size (prevent memory bombs)
+const MAX_EXTRACT_SIZE = MAX_FILE_EXTRACT_SIZE;
+const EXTRACTION_TIMEOUT_MS = FILE_EXTRACTION_TIMEOUT_MS;
+
+// File categories for routing
+const TEXT_EXTENSIONS = new Set([
+  '.txt', '.md', '.json', '.js', '.ts', '.jsx', '.tsx', '.py', '.rb',
+  '.html', '.css', '.scss', '.less', '.xml', '.yaml', '.yml', '.toml',
+  '.sh', '.bat', '.ps1', '.sql', '.csv', '.env', '.gitignore', '.log',
+  '.ini', '.cfg', '.conf', '.dockerfile', '.makefile', '.rs', '.go',
+  '.java', '.kt', '.c', '.cpp', '.h', '.hpp', '.cs', '.swift', '.r',
+  '.lua', '.php', '.pl', '.ex', '.exs', '.erl', '.hs', '.ml',
+]);
+
+const EXTRACTABLE_TYPES = {
+  'application/pdf': 'pdf',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
+  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
+  'application/vnd.ms-excel': 'xls',
+};
+
+/**
+ * Run a function with a timeout
+ */
+function withTimeout(fn, ms) {
+  return Promise.race([
+    fn(),
+    new Promise((_, reject) =>
+      setTimeout(() => reject(new Error(`Extraction timed out after ${ms}ms`)), ms)
+    ),
+  ]);
+}
+
+/**
+ * Extract text from a PDF buffer (pdf-parse v2 API)
+ */
+async function extractPdf(buffer) {
+  const PDFParse = await loadPdfParse();
+  const parser = new PDFParse({ data: buffer });
+  const result = await parser.getText();
+  const text = result?.text || '';
+  await parser.destroy();
+  return text.slice(0, MAX_EXTRACT_SIZE);
+}
+
+/**
+ * Extract text from a DOCX buffer
+ */
+async function extractDocx(buffer) {
+  const mam = await loadMammoth();
+  const result = await mam.extractRawText({ buffer });
+  return result.value?.slice(0, MAX_EXTRACT_SIZE) || '';
+}
+
+/**
+ * Extract text from an XLSX/XLS buffer
+ */
+async function extractSpreadsheet(buffer) {
+  const EJ = await loadExcelJS();
+  const workbook = new EJ.Workbook();
+  await workbook.xlsx.load(buffer);
+  const sheets = [];
+
+  workbook.eachSheet((sheet) => {
+    const rows = [];
+    sheet.eachRow((row) => {
+      const values = [];
+      row.eachCell({ includeEmpty: true }, (cell) => {
+        const val = cell.text ?? cell.value ?? '';
+        values.push(String(val).replace(/,/g, ' '));
+      });
+      rows.push(values.join(','));
+    });
+    sheets.push(`## Sheet: ${sheet.name}\n${rows.join('\n')}`);
+
+    // Check total size
+    const totalSize = sheets.join('\n\n').length;
+    if (totalSize > MAX_EXTRACT_SIZE) return;
+  });
+
+  return sheets.join('\n\n').slice(0, MAX_EXTRACT_SIZE);
+}
+
+/**
+ * Determine if a file is plain text based on extension
+ */
+function isTextByExtension(filename) {
+  const ext = path.extname(filename).toLowerCase();
+  return TEXT_EXTENSIONS.has(ext);
+}
+
+/**
+ * Setup file routes
+ * @param {Express} app - Express app instance
+ * @param {Object} deps - Dependencies
+ */
+export function setupFileRoutes(app, context) {
+  const {
+    fileUpload,
+    fetch: fetchWithTimeout,
+    log,
+    config,
+    uploadsDir,
+  } = context;
+
+  const { GATEWAY_URL, GATEWAY_TOKEN, SESSION_USER, REQUEST_TIMEOUT } = config;
+
+  app.post('/api/file', (req, res) => {
+    fileUpload(req, res, async (err) => {
+      if (err) {
+        log('error', '[File] Upload error:', err);
+        const errorDetail = process.env.NODE_ENV === 'production'
+          ? 'File upload failed'
+          : err.message;
+        return badRequest(res, 'Upload failed: ' + errorDetail, ErrorCodes.UPLOAD_FAILED);
+      }
+
+      try {
+        if (!req.file) {
+          return badRequest(res, 'No file provided', ErrorCodes.MISSING_FIELD);
+        }
+
+        const { originalname, path: tempPath, size } = req.file;
+        const caption = req.body?.caption?.slice(0, 500) || '';
+        const satelliteId = String(req.body?.satelliteId || 'main').replace(/[^a-zA-Z0-9_-]/g, '').substring(0, 32);
+
+        log('info', `[File] Received: ${originalname} (${(size / 1024).toFixed(1)} KB)`);
+
+        // Read the uploaded file
+        const fileBuffer = await fs.readFile(tempPath);
+
+        // Detect actual file type via magic bytes
+        const detectedType = await fileTypeFromBuffer(fileBuffer);
+        const mimeType = detectedType?.mime || 'application/octet-stream';
+        const ext = path.extname(originalname).toLowerCase() || (detectedType?.ext ? `.${detectedType.ext}` : '');
+
+        // Generate unique filename preserving original extension
+        const fileId = randomUUID();
+        const safeOrigName = originalname.replace(/[^a-zA-Z0-9._-]/g, '_').substring(0, 100);
+        const savedFilename = `file-${fileId}${ext}`;
+        const savedPath = path.join(uploadsDir, savedFilename);
+
+        // Save original file
+        await fs.mkdir(uploadsDir, { recursive: true });
+        await fs.writeFile(savedPath, fileBuffer);
+
+        // Clean up temp file
+        await fs.unlink(tempPath).catch(() => {});
+
+        // Attempt text extraction
+        let extractedText = null;
+        let extractedFilename = null;
+        let extractionMethod = 'none';
+
+        try {
+          if (isTextByExtension(originalname)) {
+            // Plain text — read directly
+            extractedText = fileBuffer.toString('utf-8').slice(0, MAX_EXTRACT_SIZE);
+            extractionMethod = 'text';
+          } else if (EXTRACTABLE_TYPES[mimeType] === 'pdf') {
+            extractedText = await withTimeout(() => extractPdf(fileBuffer), EXTRACTION_TIMEOUT_MS);
+            extractionMethod = 'pdf-parse';
+          } else if (EXTRACTABLE_TYPES[mimeType] === 'docx') {
+            extractedText = await withTimeout(() => extractDocx(fileBuffer), EXTRACTION_TIMEOUT_MS);
+            extractionMethod = 'mammoth';
+          } else if (EXTRACTABLE_TYPES[mimeType] === 'xlsx' || EXTRACTABLE_TYPES[mimeType] === 'xls') {
+            extractedText = await withTimeout(() => extractSpreadsheet(fileBuffer), EXTRACTION_TIMEOUT_MS);
+            extractionMethod = 'exceljs';
+          }
+        } catch (extractErr) {
+          log('warn', `[File] Text extraction failed for ${originalname}: ${extractErr.message}`);
+          // Continue without extracted text — agent can still see the file exists
+        }
+
+        // Save extracted text alongside original
+        if (extractedText && extractedText.trim().length > 0) {
+          extractedFilename = `file-${fileId}.extracted.txt`;
+          const extractedPath = path.join(uploadsDir, extractedFilename);
+          const header = `# Extracted from: ${safeOrigName}\n# Type: ${mimeType}\n# Method: ${extractionMethod}\n# Size: ${(size / 1024).toFixed(1)} KB\n---\n\n`;
+          await fs.writeFile(extractedPath, header + extractedText, 'utf-8');
+        }
+
+        // Build the message to send to the agent
+        const filePath = `uplink/uploads/${savedFilename}`;
+        const extractedPath = extractedFilename ? `uplink/uploads/${extractedFilename}` : null;
+
+        let prompt;
+        if (extractedPath) {
+          prompt = caption
+            ? `[File uploaded: ${safeOrigName}] [Original at: ${filePath}] [Extracted text at: ${extractedPath} — read this file for the content] ${caption}`
+            : `[File uploaded: ${safeOrigName}] [Original at: ${filePath}] [Extracted text at: ${extractedPath} — read this file for the content] The user sent a file. Read the extracted text and summarize or respond to its contents.`;
+        } else {
+          prompt = caption
+            ? `[File uploaded: ${safeOrigName}] [Saved at: ${filePath}] [Type: ${mimeType}, Size: ${(size / 1024).toFixed(1)} KB] ${caption}`
+            : `[File uploaded: ${safeOrigName}] [Saved at: ${filePath}] [Type: ${mimeType}, Size: ${(size / 1024).toFixed(1)} KB] The user sent a file. Acknowledge receipt and describe what you can do with it.`;
+        }
+
+        // SSE response (same pattern as image upload)
+        res.setHeader('Content-Type', 'text/event-stream; charset=utf-8');
+        res.setHeader('Cache-Control', 'no-cache');
+        res.setHeader('Connection', 'keep-alive');
+        res.setHeader('X-Accel-Buffering', 'no');
+        res.flushHeaders();
+
+        const keepalive = setInterval(() => {
+          res.write(': keepalive\n\n');
+        }, FILE_CLEANUP_DELAY_MS);
+
+        let reply = 'I received your file.';
+
+        try {
+          // Derive session key
+          const sessionKey = satelliteId === 'main'
+            ? 'agent:main:main'
+            : `agent:main:uplink:satellite:${satelliteId}`;
+
+          const response = await fetchWithTimeout(`${GATEWAY_URL}/v1/chat/completions`, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              'Authorization': `Bearer ${GATEWAY_TOKEN}`,
+              'x-openclaw-session-key': sessionKey,
+            },
+            body: JSON.stringify({
+              model: 'openclaw',
+              user: SESSION_USER,
+              messages: [{ role: 'user', content: prompt }],
+            }),
+          }, REQUEST_TIMEOUT);
+
+          if (!response.ok) {
+            const text = await response.text();
+            throw new Error(`Chat API error: ${response.status} - ${text}`);
+          }
+
+          const data = await response.json();
+          reply = data.choices?.[0]?.message?.content || reply;
+        } finally {
+          clearInterval(keepalive);
+        }
+
+        // Send result
+        res.write(`data: ${JSON.stringify({
+          response: reply,
+          filename: safeOrigName,
+          filePath,
+          extractedPath,
+          extractionMethod,
+        })}\n\n`);
+        res.end();
+
+        log('info', `[File] Processed: ${safeOrigName} (extraction: ${extractionMethod})`);
+      } catch (error) {
+        log('error', '[File] Error:', error);
+        if (res.headersSent) {
+          res.write(`data: ${JSON.stringify({ error: true, message: 'File processing failed' })}\n\n`);
+          res.end();
+        } else {
+          internalError(res, 'File processing failed', ErrorCodes.INTERNAL_ERROR);
+        }
+      }
+    });
+  });
+}

package/server/routes/index.js

@@ -0,0 +1,18 @@
+/**
+ * Routes Index - Export all route modules
+ */
+
+export { setupVoiceRoutes } from './voice.js';
+export { setupChatRoutes } from './chat.js';
+export { setupMediaRoutes } from './media.js';
+export { setupWebhookRoutes } from './webhooks.js';
+export { setupConfigRoutes } from './config.js';
+export { setupStatusRoutes } from './status.js';
+export { setupSatelliteRoutes } from './satellite.js';
+export { setupMissedMessagesRoutes, addMissedMessage } from './missed-messages.js';
+export { setupPushRoutes, sendPushNotification } from './push.js';
+export { setupSTTRoutes } from './stt.js';
+export { setupFileRoutes } from './files.js';
+export { setupAgentRoutes } from './agents.js';
+export { setupPremiumRoutes } from './premium.js';
+export { setupArtifactsRoutes } from './artifacts.js';