nothumanallowed 13.5.117 → 13.5.118

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nothumanallowed",
-  "version": "13.5.117",
+  "version": "13.5.118",
   "description": "NotHumanAllowed — 38 AI agents, 80 tools, Studio (visual agentic workflows). Email, calendar, browser automation, screen capture, canvas, cron/heartbeat, Alexandria E2E messaging, GitHub, Notion, Slack, voice chat, free AI (Liara), 28 languages. Zero-dependency CLI.",
   "type": "module",
   "bin": {

package/src/constants.mjs

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-export const VERSION = '13.5.117';
+export const VERSION = '13.5.118';
 export const BASE_URL = 'https://nothumanallowed.com/cli';
 export const API_BASE = 'https://nothumanallowed.com/api/v1';
 

package/src/services/email-imap.mjs

@@ -148,36 +148,51 @@ function stripQuotedReplies(text) {
   return text.split('\n').filter(l => !l.startsWith('>')).join('\n').trim();
 }
 
-export async function syncFolder(accountId, folderPath, fullResync) {
+export async function syncFolder(accountId, folderPath, fullResync, limitMessages = 0) {
   const client = await getImapClient(accountId);
   const db = getDb();
   const folderMeta = getFolder(accountId, folderPath);
+
+  // Open mailbox to get uidValidity BEFORE getMailboxLock
   const lock = await client.getMailboxLock(folderPath);
 
   try {
     const mailbox = client.mailbox;
     const currentUidValidity = Number(mailbox?.uidValidity ?? 0);
+    const totalMessages = Number(mailbox?.exists ?? 0);
     const storedUidValidity = folderMeta?.uid_validity ?? null;
-    const lastUid = (fullResync || storedUidValidity !== currentUidValidity) ? 0 : (folderMeta?.last_uid ?? 0);
+    const isFirstSync = storedUidValidity === null || fullResync;
+    const lastUid = isFirstSync ? 0 : (folderMeta?.last_uid ?? 0);
     const needsResync = fullResync || (storedUidValidity !== null && storedUidValidity !== currentUidValidity);
 
     if (needsResync && folderMeta) {
-      // Clear existing messages for this folder on UID validity change
       db.prepare('DELETE FROM email_message_labels WHERE message_id IN (SELECT id FROM email_messages WHERE account_id = ? AND imap_folder_path = ?)').run(accountId, folderPath);
       db.prepare('DELETE FROM email_messages WHERE account_id = ? AND imap_folder_path = ?').run(accountId, folderPath);
     }
 
     const inboxLabel = getSystemLabel(accountId, 'inbox');
     const sentLabel = getSystemLabel(accountId, 'sent');
-    const blockedLabel = getSystemLabel(accountId, 'spam');
 
     let newLastUid = lastUid;
     let synced = 0;
 
-    const range = lastUid > 0 ? `${lastUid + 1}:*` : '1:*';
+    // On first sync: fetch only the most recent N messages by sequence number
+    // On incremental sync: fetch new UIDs since last known
+    let range;
+    if (isFirstSync && limitMessages > 0 && totalMessages > limitMessages) {
+      const seqStart = totalMessages - limitMessages + 1;
+      range = `${seqStart}:*`;
+    } else if (lastUid > 0) {
+      range = `${lastUid + 1}:*`;
+    } else {
+      range = '1:*';
+    }
 
+    // Phase 1: fetch headers only (fast — no body download)
+    const headerMap = new Map(); // uid → parsed header data
     for await (const msg of client.fetch(range, {
       uid: true, flags: true, envelope: true, internalDate: true, size: true,
+      bodyStructure: true,
     }, { uid: true })) {
       if (msg.uid <= lastUid) continue;
       newLastUid = Math.max(newLastUid, msg.uid);
@@ -186,36 +201,48 @@ export async function syncFolder(accountId, folderPath, fullResync) {
       const fromAddr = env.from?.[0]?.address ?? null;
       const fromName = env.from?.[0]?.name ?? null;
 
-      // Skip blocked senders
-      if (fromAddr && isSenderBlocked(accountId, fromAddr)) {
-        synced++;
-        continue;
-      }
-
-      // Check if already synced
+      if (fromAddr && isSenderBlocked(accountId, fromAddr)) { synced++; continue; }
       if (messageExists(accountId, folderPath, msg.uid)) { synced++; continue; }
 
-      // Fetch full body for this message
+      headerMap.set(msg.uid, {
+        env, fromAddr, fromName,
+        internalDate: (msg.internalDate || new Date()).toISOString(),
+        size: msg.size || 0,
+        flags: msg.flags,
+        bodyStructure: msg.bodyStructure,
+      });
+    }
+
+    // Phase 2: for each new message, fetch body (individually, with error isolation)
+    const uidsToFetch = [...headerMap.keys()];
+    for (const uid of uidsToFetch) {
+      const hdr = headerMap.get(uid);
+      const env = hdr.env;
+      const fromAddr = hdr.fromAddr;
+      const fromName = hdr.fromName;
+
       let bodyText = null, bodyHtml = null, bodyPreview = '', attachments = [];
       let inReplyTo = null, references = [], msgId = null;
 
       try {
-        const dl = await client.download(String(msg.uid), undefined, { uid: true });
+        const dl = await client.download(String(uid), undefined, { uid: true });
         if (dl) {
           const chunks = [];
           let totalSize = 0;
           for await (const chunk of dl.content) {
             totalSize += chunk.length;
-            if (totalSize > 25 * 1024 * 1024) break; // 25MB cap
+            if (totalSize > 10 * 1024 * 1024) break; // 10MB cap per message
             chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
           }
           const raw = Buffer.concat(chunks);
-          const parsed = await simpleParser(raw);
+          const parsed = await simpleParser(raw, { skipHtmlToText: false });
           msgId = parsed.messageId || null;
-          inReplyTo = typeof parsed.inReplyTo === 'string' ? parsed.inReplyTo : (Array.isArray(parsed.inReplyTo) ? parsed.inReplyTo[0] : null);
-          references = Array.isArray(parsed.references) ? parsed.references.slice(0, 20) : (parsed.references ? [parsed.references] : []);
-          bodyText = parsed.text?.replace(/\x00/g, '') || null;
-          bodyHtml = parsed.html?.replace(/\x00/g, '') || null;
+          inReplyTo = typeof parsed.inReplyTo === 'string' ? parsed.inReplyTo
+            : (Array.isArray(parsed.inReplyTo) ? parsed.inReplyTo[0] : null);
+          references = Array.isArray(parsed.references) ? parsed.references.slice(0, 20)
+            : (parsed.references ? [parsed.references] : []);
+          bodyText = (typeof parsed.text === 'string' ? parsed.text : null)?.replace(/\x00/g, '') || null;
+          bodyHtml = (typeof parsed.html === 'string' ? parsed.html : null)?.replace(/\x00/g, '') || null;
           bodyPreview = (bodyText || '').replace(/\s+/g, ' ').trim().slice(0, 255);
 
           if (parsed.attachments) {
@@ -226,30 +253,31 @@ export async function syncFolder(accountId, folderPath, fullResync) {
                 size_bytes: att.size || 0,
                 part_id: att.partId || '',
                 content_id: att.contentId || null,
-                content: att.size < 5 * 1024 * 1024 ? att.content : null,
+                content: (att.size || 0) < 5 * 1024 * 1024 ? att.content : null,
               });
             }
           }
         }
       } catch (err) {
-        console.warn(`[email:imap] Failed to fetch body uid=${msg.uid}:`, err.message);
+        console.warn(`[email:imap] Failed to fetch body uid=${uid}:`, err.message);
+        // Continue with headers-only for this message
       }
 
       const toAddresses = (env.to || []).map(a => ({ address: a.address, name: a.name }));
       const ccAddresses = (env.cc || []).map(a => ({ address: a.address, name: a.name }));
       const bccAddresses = (env.bcc || []).map(a => ({ address: a.address, name: a.name }));
       const subject = env.subject || '';
-      const internalDate = (msg.internalDate || new Date()).toISOString();
-      const tid = threadId(msgId, inReplyTo, references, fromAddr, internalDate, subject);
-      const hash = contentHash(msgId || '', fromAddr || '', internalDate);
+      const tid = threadId(msgId, inReplyTo, references, fromAddr, hdr.internalDate, subject);
+      const hash = contentHash(msgId || '', fromAddr || '', hdr.internalDate);
 
-      const folderRec = upsertFolder(accountId, folderPath, folderPath, folderMeta?.folder_type || 'custom', currentUidValidity, newLastUid);
+      const folderRec = upsertFolder(accountId, folderPath, folderPath,
+        folderMeta?.folder_type || 'custom', currentUidValidity, newLastUid);
 
       const msgDbId = insertMessage({
         account_id: accountId,
         folder_id: folderRec,
         imap_folder_path: folderPath,
-        uid: msg.uid,
+        uid,
         message_id: msgId,
         in_reply_to: inReplyTo,
         references_list: references,
@@ -264,42 +292,41 @@ export async function syncFolder(accountId, folderPath, fullResync) {
         body_html: bodyHtml,
         body_preview: bodyPreview,
         body_reply_only: stripQuotedReplies(bodyText),
-        size_bytes: msg.size || 0,
+        size_bytes: hdr.size,
         has_attachments: attachments.length > 0,
         content_hash: hash,
-        internal_date: internalDate,
+        internal_date: hdr.internalDate,
         source: 'imap',
       });
 
       if (attachments.length > 0) insertAttachments(msgDbId, attachments);
 
-      // Apply archiving rules (first match wins, removes from inbox)
       const archived = applyArchivingRules(accountId, msgDbId, fromAddr, subject);
       if (!archived) {
-        // Assign label based on folder type
-        const isSent = folderPath.toLowerCase().includes('sent') || folderPath.toLowerCase().includes('inviati');
-        const targetLabel = isSent ? sentLabel : inboxLabel;
+        const isSentFolder = folderPath.toLowerCase().includes('sent') || folderPath.toLowerCase().includes('inviati');
+        const targetLabel = isSentFolder ? sentLabel : inboxLabel;
         if (targetLabel) addMessageToLabel(msgDbId, targetLabel.id);
       }
 
       synced++;
     }
 
-    // Update folder sync state
-    upsertFolder(accountId, folderPath,
-      folderPath, folderMeta?.folder_type || 'custom',
-      currentUidValidity, newLastUid);
+    upsertFolder(accountId, folderPath, folderPath,
+      folderMeta?.folder_type || 'custom', currentUidValidity, newLastUid);
 
-    return { synced, lastUid: newLastUid };
+    updateLabelCounts(accountId);
+    return { synced, lastUid: newLastUid, total: totalMessages };
   } finally {
     lock.release();
   }
 }
 
-export async function syncAccount(accountId) {
+// First sync: cap to last 200 messages per folder to avoid blocking for minutes
+const FIRST_SYNC_LIMIT = 200;
+
+export async function syncAccount(accountId, opts = {}) {
   setSyncStatus(accountId, 'syncing', null);
   try {
-    // List folders from IMAP and sync INBOX + Sent
     const folders = await listImapFolders(accountId);
     const priority = ['inbox', 'sent'];
     const toSync = [
@@ -310,8 +337,10 @@ export async function syncAccount(accountId) {
     let totalSynced = 0;
     for (const f of toSync) {
       try {
-        const result = await syncFolder(accountId, f.path, false);
+        const limit = opts.full ? 0 : FIRST_SYNC_LIMIT;
+        const result = await syncFolder(accountId, f.path, false, limit);
         totalSynced += result.synced;
+        console.log(`[email:sync] ${f.path}: ${result.synced} new messages (total on server: ${result.total})`);
       } catch (err) {
         console.warn(`[email:imap] Sync folder ${f.path} failed:`, err.message);
       }