nothumanallowed 13.5.113 → 13.5.115

package/src/services/email-imap.mjs

@@ -0,0 +1,421 @@
+/**
+ * email-imap.mjs — IMAP sync service (READ-ONLY)
+ *
+ * ⛔ IMAP IS STRICTLY READ-ONLY.
+ * This service NEVER writes to the IMAP server:
+ * - No setFlags() — read/starred state is in SQLite only
+ * - No moveMessage() — label moves are in SQLite only
+ * - No DELETE — emails are never deleted from the server
+ *
+ * Handles: connection pool, incremental sync, IDLE push, folder listing.
+ */
+
+import { ImapFlow } from 'imapflow';
+import { simpleParser } from 'mailparser';
+import { createHash } from 'crypto';
+import {
+  getAccountCredentials, setSyncStatus, upsertFolder, getFolder,
+  updateFolderUid, insertMessage, insertAttachments, addMessageToLabel,
+  getSystemLabel, isSenderBlocked, applyArchivingRules, messageExists,
+  updateLabelCounts, getDb, contentHash,
+} from './email-db.mjs';
+
+// ── FOLDER TYPE MAPPING ────────────────────────────────────────────────────
+
+const SPECIAL_USE_MAP = {
+  '\\Inbox': 'inbox', '\\Sent': 'sent', '\\Drafts': 'drafts',
+  '\\Junk': 'spam', '\\Trash': 'trash',
+};
+
+const FOLDER_NAME_MAP = {
+  'inbox': 'inbox', 'sent': 'sent', 'sent messages': 'sent', 'sent items': 'sent',
+  'posta inviata': 'sent', 'inviati': 'sent', 'drafts': 'drafts', 'bozze': 'drafts',
+  'draft': 'drafts', 'spam': 'spam', 'junk': 'spam', 'junk e-mail': 'spam',
+  'posta indesiderata': 'spam', 'trash': 'trash', 'cestino': 'trash',
+  'deleted': 'trash', 'deleted messages': 'trash', 'deleted items': 'trash',
+};
+
+// ── CONNECTION POOL ────────────────────────────────────────────────────────
+
+const syncClients = new Map();   // accountId → ImapFlow (sync)
+const idleClients = new Map();   // accountId → ImapFlow (IDLE)
+const idleTimers = new Map();
+const heartbeatTimers = new Map();
+const lastIdleEvents = new Map();
+const reconnectAttempts = new Map();
+
+const IDLE_RESTART_MS = 25 * 60 * 1000;         // 25 min
+const IDLE_HEARTBEAT_WATCHDOG_MS = 12 * 60 * 1000; // 12 min
+const RECONNECT_DELAYS = [5000, 15000, 30000, 60000, 120000, 300000];
+
+const idleHandlers = new Set();
+export function onIdleNotification(handler) {
+  idleHandlers.add(handler);
+  return () => idleHandlers.delete(handler);
+}
+function notifyIdle(accountId, folder) {
+  for (const h of idleHandlers) { try { h(accountId, folder); } catch {} }
+}
+
+function createImapClient(label, creds, accountId) {
+  const isSecure = creds.imap_port === 993;
+  const client = new ImapFlow({
+    host: creds.imap_host,
+    port: creds.imap_port,
+    secure: isSecure,
+    auth: { user: creds.username, pass: creds.password },
+    logger: false,
+    clientInfo: { name: 'NHA-Mail', version: '1.0.0' },
+    emitLogs: false,
+    ...(!isSecure && { tls: { rejectUnauthorized: false } }),
+  });
+  client.on('error', (err) => {
+    console.error(`[email:imap] ${label} error:`, err.message);
+    if (accountId) syncClients.delete(accountId);
+  });
+  return client;
+}
+
+export async function getImapClient(accountId) {
+  const existing = syncClients.get(accountId);
+  if (existing?.usable) return existing;
+  if (existing) { syncClients.delete(accountId); try { await existing.logout(); } catch {} }
+
+  const creds = getAccountCredentials(accountId);
+  if (!creds || !creds.imap_host) throw new Error(`No IMAP credentials for account ${accountId}`);
+  const client = createImapClient(`sync:${accountId.slice(0, 8)}`, creds, accountId);
+  await client.connect();
+  syncClients.set(accountId, client);
+  return client;
+}
+
+export async function closeImapClient(accountId) {
+  const c = syncClients.get(accountId);
+  if (c) { try { await c.logout(); } catch {}; syncClients.delete(accountId); }
+}
+
+// ── FOLDER LISTING ─────────────────────────────────────────────────────────
+
+export async function listImapFolders(accountId) {
+  const client = await getImapClient(accountId);
+  const tree = await client.listTree();
+  const all = [];
+
+  function walk(items) {
+    if (!items) return;
+    for (const item of items) {
+      const name = item.name ?? '';
+      const path = item.path ?? '';
+      let folderType = 'custom';
+      if (item.specialUse) folderType = SPECIAL_USE_MAP[item.specialUse] ?? 'custom';
+      if (folderType === 'custom') folderType = FOLDER_NAME_MAP[name.toLowerCase()] ?? 'custom';
+      if (path === 'INBOX') folderType = 'inbox';
+      if (!item.flags?.has('\\Noselect')) {
+        all.push({ path, name, folderType, hasSpecialUse: !!item.specialUse });
+      }
+      if (item.folders?.length) walk(item.folders);
+    }
+  }
+
+  if (tree.path === 'INBOX' || !tree.path) {
+    all.push({ path: 'INBOX', name: 'Inbox', folderType: 'inbox', hasSpecialUse: true });
+  }
+  if (tree.folders?.length) walk(tree.folders);
+
+  // Dedup: keep one winner per system type
+  const winners = new Map();
+  for (const f of all) {
+    if (f.folderType === 'custom') continue;
+    const ex = winners.get(f.folderType);
+    if (!ex || (f.hasSpecialUse && !ex.hasSpecialUse) || (!f.path.includes('.') && ex.path.includes('.'))) {
+      winners.set(f.folderType, f);
+    }
+  }
+  return all.map(f => f.folderType === 'custom' ? f : (winners.get(f.folderType) === f ? f : { ...f, folderType: 'custom' }));
+}
+
+// ── INCREMENTAL SYNC ───────────────────────────────────────────────────────
+
+function threadId(messageId, inReplyTo, references, fromAddress, date, subject) {
+  const root = (references && references[0]) || inReplyTo || messageId;
+  if (root) return createHash('sha1').update(root).digest('hex');
+  return createHash('sha1').update(`${fromAddress}|${date}|${subject}`).digest('hex');
+}
+
+function stripQuotedReplies(text) {
+  if (!text) return text;
+  return text.split('\n').filter(l => !l.startsWith('>')).join('\n').trim();
+}
+
+export async function syncFolder(accountId, folderPath, fullResync) {
+  const client = await getImapClient(accountId);
+  const db = getDb();
+  const folderMeta = getFolder(accountId, folderPath);
+  const lock = await client.getMailboxLock(folderPath);
+
+  try {
+    const mailbox = client.mailbox;
+    const currentUidValidity = Number(mailbox?.uidValidity ?? 0);
+    const storedUidValidity = folderMeta?.uid_validity ?? null;
+    const lastUid = (fullResync || storedUidValidity !== currentUidValidity) ? 0 : (folderMeta?.last_uid ?? 0);
+    const needsResync = fullResync || (storedUidValidity !== null && storedUidValidity !== currentUidValidity);
+
+    if (needsResync && folderMeta) {
+      // Clear existing messages for this folder on UID validity change
+      db.prepare('DELETE FROM email_message_labels WHERE message_id IN (SELECT id FROM email_messages WHERE account_id = ? AND imap_folder_path = ?)').run(accountId, folderPath);
+      db.prepare('DELETE FROM email_messages WHERE account_id = ? AND imap_folder_path = ?').run(accountId, folderPath);
+    }
+
+    const inboxLabel = getSystemLabel(accountId, 'inbox');
+    const sentLabel = getSystemLabel(accountId, 'sent');
+    const blockedLabel = getSystemLabel(accountId, 'spam');
+
+    let newLastUid = lastUid;
+    let synced = 0;
+
+    const range = lastUid > 0 ? `${lastUid + 1}:*` : '1:*';
+
+    for await (const msg of client.fetch(range, {
+      uid: true, flags: true, envelope: true, internalDate: true, size: true,
+    }, { uid: true })) {
+      if (msg.uid <= lastUid) continue;
+      newLastUid = Math.max(newLastUid, msg.uid);
+
+      const env = msg.envelope ?? {};
+      const fromAddr = env.from?.[0]?.address ?? null;
+      const fromName = env.from?.[0]?.name ?? null;
+
+      // Skip blocked senders
+      if (fromAddr && isSenderBlocked(accountId, fromAddr)) {
+        synced++;
+        continue;
+      }
+
+      // Check if already synced
+      if (messageExists(accountId, folderPath, msg.uid)) { synced++; continue; }
+
+      // Fetch full body for this message
+      let bodyText = null, bodyHtml = null, bodyPreview = '', attachments = [];
+      let inReplyTo = null, references = [], msgId = null;
+
+      try {
+        const dl = await client.download(String(msg.uid), undefined, { uid: true });
+        if (dl) {
+          const chunks = [];
+          let totalSize = 0;
+          for await (const chunk of dl.content) {
+            totalSize += chunk.length;
+            if (totalSize > 25 * 1024 * 1024) break; // 25MB cap
+            chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+          }
+          const raw = Buffer.concat(chunks);
+          const parsed = await simpleParser(raw);
+          msgId = parsed.messageId || null;
+          inReplyTo = typeof parsed.inReplyTo === 'string' ? parsed.inReplyTo : (Array.isArray(parsed.inReplyTo) ? parsed.inReplyTo[0] : null);
+          references = Array.isArray(parsed.references) ? parsed.references.slice(0, 20) : (parsed.references ? [parsed.references] : []);
+          bodyText = parsed.text?.replace(/\x00/g, '') || null;
+          bodyHtml = parsed.html?.replace(/\x00/g, '') || null;
+          bodyPreview = (bodyText || '').replace(/\s+/g, ' ').trim().slice(0, 255);
+
+          if (parsed.attachments) {
+            for (const att of parsed.attachments) {
+              attachments.push({
+                filename: att.filename || null,
+                content_type: att.contentType || 'application/octet-stream',
+                size_bytes: att.size || 0,
+                part_id: att.partId || '',
+                content_id: att.contentId || null,
+                content: att.size < 5 * 1024 * 1024 ? att.content : null,
+              });
+            }
+          }
+        }
+      } catch (err) {
+        console.warn(`[email:imap] Failed to fetch body uid=${msg.uid}:`, err.message);
+      }
+
+      const toAddresses = (env.to || []).map(a => ({ address: a.address, name: a.name }));
+      const ccAddresses = (env.cc || []).map(a => ({ address: a.address, name: a.name }));
+      const bccAddresses = (env.bcc || []).map(a => ({ address: a.address, name: a.name }));
+      const subject = env.subject || '';
+      const internalDate = (msg.internalDate || new Date()).toISOString();
+      const tid = threadId(msgId, inReplyTo, references, fromAddr, internalDate, subject);
+      const hash = contentHash(msgId || '', fromAddr || '', internalDate);
+
+      const folderRec = upsertFolder(accountId, folderPath, folderPath, folderMeta?.folder_type || 'custom', currentUidValidity, newLastUid);
+
+      const msgDbId = insertMessage({
+        account_id: accountId,
+        folder_id: folderRec,
+        imap_folder_path: folderPath,
+        uid: msg.uid,
+        message_id: msgId,
+        in_reply_to: inReplyTo,
+        references_list: references,
+        thread_id: tid,
+        subject,
+        from_address: fromAddr,
+        from_name: fromName,
+        to_addresses: toAddresses,
+        cc_addresses: ccAddresses,
+        bcc_addresses: bccAddresses,
+        body_text: bodyText,
+        body_html: bodyHtml,
+        body_preview: bodyPreview,
+        body_reply_only: stripQuotedReplies(bodyText),
+        size_bytes: msg.size || 0,
+        has_attachments: attachments.length > 0,
+        content_hash: hash,
+        internal_date: internalDate,
+        source: 'imap',
+      });
+
+      if (attachments.length > 0) insertAttachments(msgDbId, attachments);
+
+      // Apply archiving rules (first match wins, removes from inbox)
+      const archived = applyArchivingRules(accountId, msgDbId, fromAddr, subject);
+      if (!archived) {
+        // Assign label based on folder type
+        const isSent = folderPath.toLowerCase().includes('sent') || folderPath.toLowerCase().includes('inviati');
+        const targetLabel = isSent ? sentLabel : inboxLabel;
+        if (targetLabel) addMessageToLabel(msgDbId, targetLabel.id);
+      }
+
+      synced++;
+    }
+
+    // Update folder sync state
+    upsertFolder(accountId, folderPath,
+      folderPath, folderMeta?.folder_type || 'custom',
+      currentUidValidity, newLastUid);
+
+    return { synced, lastUid: newLastUid };
+  } finally {
+    lock.release();
+  }
+}
+
+export async function syncAccount(accountId) {
+  setSyncStatus(accountId, 'syncing', null);
+  try {
+    // List folders from IMAP and sync INBOX + Sent
+    const folders = await listImapFolders(accountId);
+    const priority = ['inbox', 'sent'];
+    const toSync = [
+      ...folders.filter(f => priority.includes(f.folderType)),
+      ...folders.filter(f => !priority.includes(f.folderType) && f.folderType !== 'trash'),
+    ];
+
+    let totalSynced = 0;
+    for (const f of toSync) {
+      try {
+        const result = await syncFolder(accountId, f.path, false);
+        totalSynced += result.synced;
+      } catch (err) {
+        console.warn(`[email:imap] Sync folder ${f.path} failed:`, err.message);
+      }
+    }
+    setSyncStatus(accountId, 'idle', null);
+    return { synced: totalSynced };
+  } catch (err) {
+    setSyncStatus(accountId, 'error', err.message);
+    throw err;
+  }
+}
+
+// ── IDLE (push notifications for new messages) ─────────────────────────────
+
+export async function startIdle(accountId) {
+  await connectIdle(accountId);
+}
+
+async function connectIdle(accountId) {
+  const creds = getAccountCredentials(accountId);
+  if (!creds?.imap_host) return;
+
+  try {
+    const existing = idleClients.get(accountId);
+    if (existing) { try { await existing.logout(); } catch {} }
+
+    const client = createImapClient(`idle:${accountId.slice(0, 8)}`, creds);
+    await client.connect();
+    idleClients.set(accountId, client);
+    reconnectAttempts.set(accountId, 0);
+    lastIdleEvents.set(accountId, Date.now());
+
+    runIdleLoop(accountId).catch(() => scheduleReconnect(accountId));
+  } catch (err) {
+    console.error(`[email:idle] Connect failed for ${accountId.slice(0, 8)}:`, err.message);
+    scheduleReconnect(accountId);
+  }
+}
+
+async function runIdleLoop(accountId) {
+  const client = idleClients.get(accountId);
+  if (!client?.usable) return;
+  const lock = await client.getMailboxLock('INBOX');
+  try {
+    client.on('exists', (data) => {
+      lastIdleEvents.set(accountId, Date.now());
+      if (data.count > data.prevCount) notifyIdle(accountId, data.path);
+    });
+
+    const restart = async () => {
+      clearTimeout(idleTimers.get(accountId));
+      clearTimeout(heartbeatTimers.get(accountId));
+      const c = idleClients.get(accountId);
+      if (!c?.usable) { scheduleReconnect(accountId); return; }
+      lastIdleEvents.set(accountId, Date.now());
+      heartbeatTimers.set(accountId, setTimeout(() => {
+        if (Date.now() - (lastIdleEvents.get(accountId) ?? 0) > IDLE_HEARTBEAT_WATCHDOG_MS) {
+          scheduleReconnect(accountId);
+        }
+      }, IDLE_HEARTBEAT_WATCHDOG_MS));
+      idleTimers.set(accountId, setTimeout(async () => {
+        try { await restart(); } catch { scheduleReconnect(accountId); }
+      }, IDLE_RESTART_MS));
+      await c.idle();
+    };
+    await restart();
+  } finally {
+    lock.release();
+  }
+}
+
+function scheduleReconnect(accountId) {
+  clearTimeout(idleTimers.get(accountId));
+  clearTimeout(heartbeatTimers.get(accountId));
+  idleTimers.delete(accountId); heartbeatTimers.delete(accountId);
+  const attempt = reconnectAttempts.get(accountId) ?? 0;
+  const delay = RECONNECT_DELAYS[Math.min(attempt, RECONNECT_DELAYS.length - 1)];
+  reconnectAttempts.set(accountId, attempt + 1);
+  setTimeout(() => connectIdle(accountId).catch(() => scheduleReconnect(accountId)), delay);
+}
+
+export async function stopIdle(accountId) {
+  clearTimeout(idleTimers.get(accountId));
+  clearTimeout(heartbeatTimers.get(accountId));
+  idleTimers.delete(accountId); heartbeatTimers.delete(accountId);
+  const c = idleClients.get(accountId);
+  if (c) { try { await c.logout(); } catch {}; idleClients.delete(accountId); }
+}
+
+export async function stopAllIdle() {
+  for (const id of [...idleClients.keys()]) await stopIdle(id);
+}
+
+// ── ATTACHMENT FETCH ───────────────────────────────────────────────────────
+
+export async function fetchAttachmentContent(accountId, folderPath, uid, partId) {
+  const client = await getImapClient(accountId);
+  const lock = await client.getMailboxLock(folderPath);
+  try {
+    const dl = await client.download(String(uid), partId, { uid: true });
+    if (!dl) return null;
+    const chunks = [];
+    for await (const chunk of dl.content) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    return { buffer: Buffer.concat(chunks), contentType: dl.meta?.contentType };
+  } finally {
+    lock.release();
+  }
+}

package/src/services/email-smtp.mjs

@@ -0,0 +1,135 @@
+/**
+ * email-smtp.mjs — SMTP send service
+ *
+ * Nodemailer-based. Fresh transporter per send (avoids stale connection issues).
+ * Handles: plain send, reply/forward with threading headers, attachments.
+ */
+
+import { createTransport } from 'nodemailer';
+import { randomUUID } from 'crypto';
+import {
+  getAccountCredentials, insertMessage, addMessageToLabel, getSystemLabel,
+  insertAttachments, getDb,
+} from './email-db.mjs';
+import { createHash } from 'crypto';
+
+function threadId(messageId) {
+  if (!messageId) return createHash('sha1').update(randomUUID()).digest('hex');
+  return createHash('sha1').update(messageId).digest('hex');
+}
+
+function getTransporter(creds) {
+  const isSecure = creds.smtp_port === 465;
+  return createTransport({
+    host: creds.smtp_host,
+    port: creds.smtp_port,
+    secure: isSecure,
+    auth: { user: creds.username, pass: creds.password },
+    connectionTimeout: 15000,
+    greetingTimeout: 10000,
+    socketTimeout: 20000,
+    tls: { rejectUnauthorized: isSecure || creds.smtp_port === 587 },
+  });
+}
+
+/**
+ * Send an email.
+ *
+ * @param {string} accountId
+ * @param {object} opts
+ *   - to: string | string[]
+ *   - cc: string | string[] (optional)
+ *   - bcc: string | string[] (optional)
+ *   - subject: string
+ *   - bodyHtml: string
+ *   - bodyText: string (optional, auto-generated from html if omitted)
+ *   - inReplyTo: string (Message-ID of original, for replies)
+ *   - references: string[] (for threading)
+ *   - attachments: [{filename, content: Buffer|string, contentType}]
+ *   - replyToMessageDbId: string (DB id of original message, for label assignment)
+ */
+export async function sendEmail(accountId, opts) {
+  const creds = getAccountCredentials(accountId);
+  if (!creds?.smtp_host) throw new Error('No SMTP credentials configured for this account');
+
+  const msgId = `<${randomUUID()}@nha.local>`;
+  const fromLine = creds.from_name ? `${creds.from_name} <${creds.email_address}>` : creds.email_address;
+
+  const mailOpts = {
+    messageId: msgId,
+    from: fromLine,
+    to: Array.isArray(opts.to) ? opts.to.join(', ') : opts.to,
+    subject: opts.subject || '',
+    html: opts.bodyHtml || '',
+    text: opts.bodyText || stripHtml(opts.bodyHtml || ''),
+    envelope: { from: creds.email_address, to: toArray(opts.to) },
+  };
+
+  if (opts.cc?.length)         mailOpts.cc  = Array.isArray(opts.cc)  ? opts.cc.join(', ')  : opts.cc;
+  if (opts.bcc?.length)        mailOpts.bcc = Array.isArray(opts.bcc) ? opts.bcc.join(', ') : opts.bcc;
+  if (opts.inReplyTo)          mailOpts['In-Reply-To'] = opts.inReplyTo;
+  if (opts.references?.length) mailOpts['References']  = opts.references.join(' ');
+
+  if (opts.attachments?.length) {
+    mailOpts.attachments = opts.attachments.map(a => ({
+      filename: a.filename,
+      content:  a.content,
+      contentType: a.contentType || 'application/octet-stream',
+    }));
+  }
+
+  const transporter = getTransporter(creds);
+  let lastError;
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      await transporter.sendMail(mailOpts);
+      break;
+    } catch (err) {
+      lastError = err;
+      if (attempt < 2) await new Promise(r => setTimeout(r, 2000));
+    }
+  }
+  if (lastError) throw lastError;
+
+  // ── Save to local DB as "sent" ────────────────────────────────────────
+  const now = new Date().toISOString();
+  const tid = threadId(opts.inReplyTo || msgId);
+  const sentLabel = getSystemLabel(accountId, 'sent');
+
+  const dbId = insertMessage({
+    account_id: accountId,
+    imap_folder_path: 'Sent',
+    uid: Date.now(), // synthetic UID for sent messages
+    message_id: msgId,
+    in_reply_to: opts.inReplyTo || null,
+    references_list: opts.references || [],
+    thread_id: tid,
+    subject: opts.subject || '',
+    from_address: creds.email_address,
+    from_name: creds.from_name || null,
+    to_addresses: toArray(opts.to).map(a => ({ address: a, name: '' })),
+    cc_addresses: toArray(opts.cc).map(a => ({ address: a, name: '' })),
+    bcc_addresses: toArray(opts.bcc).map(a => ({ address: a, name: '' })),
+    body_html: opts.bodyHtml || null,
+    body_text: opts.bodyText || null,
+    body_preview: (opts.bodyText || stripHtml(opts.bodyHtml || '')).slice(0, 255),
+    size_bytes: (opts.bodyHtml || '').length,
+    has_attachments: (opts.attachments?.length || 0) > 0,
+    internal_date: now,
+    source: 'sent',
+  });
+
+  if (sentLabel) addMessageToLabel(dbId, sentLabel.id);
+
+  return { messageId: msgId, dbId };
+}
+
+function toArray(v) {
+  if (!v) return [];
+  if (Array.isArray(v)) return v.map(x => typeof x === 'string' ? x : x.address || x.email || '').filter(Boolean);
+  return v.split(',').map(s => s.trim()).filter(Boolean);
+}
+
+function stripHtml(html) {
+  return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+}

package/src/services/llm.mjs

@@ -8,10 +8,15 @@
 // ── Providers ──────────────────────────────────────────────────────────────
 
 export async function callAnthropic(apiKey, model, systemPrompt, userMessage, stream = false, opts = {}) {
+  // Use Anthropic prompt caching: system prompt as array with cache_control
+  // so the same system prompt is served from cache on repeated calls (~90% saving on input tokens).
+  const systemBlocks = systemPrompt
+    ? [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }]
+    : [];
   const body = {
     model: model || 'claude-sonnet-4-20250514',
     max_tokens: opts.max_tokens || 8192,
-    system: systemPrompt,
+    system: systemBlocks,
     messages: [{ role: 'user', content: userMessage }],
     stream,
   };
@@ -22,6 +27,7 @@ export async function callAnthropic(apiKey, model, systemPrompt, userMessage, st
       'Content-Type': 'application/json',
       'x-api-key': apiKey,
       'anthropic-version': '2023-06-01',
+      'anthropic-beta': 'prompt-caching-2024-07-31',
     },
     body: JSON.stringify(body),
   });