@adaptic/maestro 1.0.0

package/scripts/poller/gmail-poller.mjs

@@ -0,0 +1,156 @@
+// Gmail Poller — polls Sophie's inbox via native Node.js IMAP.
+// Uses imapflow + mailparser to fetch unseen emails from sophie@adaptic.ai
+// and write them as .json files to state/inbox/gmail/.
+//
+// Previously depended on scripts/sophie-inbox-poller.py (now retired).
+// All IMAP logic is now in imap-client.mjs, shared with Mehran's poller.
+
+import { readFileSync, readdirSync, renameSync } from "fs";
+import { join } from "path";
+import { readCursor, writeCursor, SOPHIE_AI_DIR } from "./utils.mjs";
+import { pollImapInbox } from "./imap-client.mjs";
+
+const GMAIL_INBOX_DIR = join(SOPHIE_AI_DIR, "state", "inbox", "gmail");
+const MAX_ITEMS_PER_POLL = 10;
+
+/**
+ * Load Sophie's Gmail app password from env or .env file.
+ */
+function loadSophiePassword() {
+  if (process.env.GMAIL_APP_PASSWORD) return process.env.GMAIL_APP_PASSWORD;
+  try {
+    const envFile = readFileSync(join(SOPHIE_AI_DIR, ".env"), "utf-8");
+    for (const line of envFile.split("\n")) {
+      if (line.startsWith("GMAIL_APP_PASSWORD=")) {
+        let val = line.split("=", 2)[1].trim();
+        if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
+          val = val.slice(1, -1);
+        }
+        return val;
+      }
+    }
+  } catch { /* no .env file */ }
+  return "";
+}
+
+/**
+ * Extract sender name from email "From" header.
+ */
+function extractSenderName(from) {
+  const match = from.match(/^"?([^"<]+)"?\s*</);
+  return match ? match[1].trim() : from.trim();
+}
+
+/**
+ * Extract sender email from "From" header.
+ */
+function extractSenderEmail(from) {
+  const match = from.match(/<([^>]+)>/);
+  return match ? match[1].trim() : from.trim();
+}
+
+/**
+ * Determine sender privilege from email address.
+ */
+function resolveSenderPrivilege(email) {
+  const lower = email.toLowerCase();
+  if (lower.includes("mehran")) return "ceo";
+  if (lower.includes("adaptic.ai")) return "internal";
+  return "external";
+}
+
+export async function pollGmail() {
+  const items = [];
+  const errors = [];
+
+  // Run native IMAP poll for sophie@adaptic.ai
+  try {
+    const result = await pollImapInbox({
+      email: "sophie@adaptic.ai",
+      password: loadSophiePassword(),
+      account: "sophie",
+      fileSuffix: "email",
+      eventType: "sophie_email",
+      logPrefix: "[sophie-gmail]",
+    });
+    if (result.errors.length > 0) {
+      errors.push(...result.errors);
+    }
+  } catch (err) {
+    const msg = `Sophie Gmail IMAP poll failed: ${err.message}`;
+    console.error(`[gmail] ${msg}`);
+    errors.push(msg);
+  }
+
+  // Read resulting JSON files (same as before — the IMAP client writes them)
+  try {
+    const files = readdirSync(GMAIL_INBOX_DIR)
+      .filter((f) => f.endsWith("-email.json") && !f.endsWith(".processed"))
+      .sort()
+      .slice(0, MAX_ITEMS_PER_POLL);
+
+    for (const file of files) {
+      try {
+        const raw = readFileSync(join(GMAIL_INBOX_DIR, file), "utf-8");
+        const data = JSON.parse(raw);
+        const email = data.email || {};
+        const senderEmail = extractSenderEmail(email.from || "unknown");
+        const senderName = extractSenderName(email.from || "unknown");
+        const privilege = resolveSenderPrivilege(senderEmail);
+        const id = file.replace(/-email\.json$/, "");
+
+        // Pass through attachment metadata
+        const attachments = data.attachments
+          ? data.attachments.map((a) => ({
+              name: a.name || "unnamed",
+              mimetype: a.mimetype || "unknown",
+              size: a.size || 0,
+              local_path: a.local_path || null,
+            }))
+          : undefined;
+
+        items.push({
+          id,
+          service: "gmail",
+          channel: `email/${senderEmail}`,
+          sender: senderName,
+          sender_email: senderEmail,
+          sender_privilege: privilege,
+          timestamp: data.received_at || email.date || new Date().toISOString(),
+          subject: (email.subject || "").replace(/\r\n/g, " ").trim(),
+          content: `Email from ${email.from || "unknown"}\nSubject: ${email.subject || "(no subject)"}\nDate: ${email.date || "unknown"}`,
+          thread_id: email.message_id || "",
+          is_reply: (email.subject || "").toLowerCase().startsWith("re:"),
+          attachments,
+          priority_signals: {
+            from_ceo: privilege === "ceo",
+            tagged_urgent: /urgent|asap|critical/i.test(email.subject || ""),
+            contains_deadline: /deadline|due|by\s+(monday|tuesday|wednesday|thursday|friday|tomorrow|end of)/i.test(email.subject || ""),
+            mentions_sophie: /sophie/i.test(email.subject || ""),
+          },
+          raw_ref: `gmail:${id}`,
+          _inbox_file: file,
+        });
+
+        // Mark as processed after reading
+        try {
+          renameSync(
+            join(GMAIL_INBOX_DIR, file),
+            join(GMAIL_INBOX_DIR, file + ".processed"),
+          );
+        } catch {
+          // Already processed
+        }
+      } catch (err) {
+        errors.push(`Gmail file ${file}: ${err.message}`);
+      }
+    }
+  } catch (err) {
+    if (err.code !== "ENOENT") {
+      errors.push(`Gmail inbox scan: ${err.message}`);
+    }
+  }
+
+  writeCursor("gmail", new Date().toISOString());
+  return { items, errors };
+}

package/scripts/poller/imap-client.mjs

@@ -0,0 +1,286 @@
+// imap-client.mjs — Shared native IMAP polling for Sophie and Mehran inboxes
+// Replaces external Python scripts with direct Node.js IMAP via imapflow.
+//
+// Each inbox gets its own credentials and file-naming convention, but the
+// connect → search UNSEEN → fetch headers+body → write JSON → disconnect
+// flow is identical and lives here.
+
+import { ImapFlow } from "imapflow";
+import { writeFileSync, existsSync, readFileSync, mkdirSync } from "fs";
+import { join, dirname } from "path";
+import { createHash } from "crypto";
+import { SOPHIE_AI_DIR } from "./utils.mjs";
+
+const INBOX_DIR = join(SOPHIE_AI_DIR, "state", "inbox", "gmail");
+const ATTACHMENTS_DIR = join(SOPHIE_AI_DIR, "state", "inbox", "attachments");
+const MAX_EMAILS_PER_CYCLE = 20;
+const SNIPPET_LENGTH = 500;
+const IMAP_TIMEOUT_MS = 25_000;
+
+// ── Helpers ─────────────────────────────────────────────────────────────
+
+function messageIdHash(messageId, fromAddr = "", subject = "", date = "") {
+  const key = messageId || `${fromAddr}${subject}${date}`;
+  return createHash("md5").update(key).digest("hex").slice(0, 16);
+}
+
+function stripHtml(html) {
+  let text = html.replace(/<(style|script)[^>]*>[\s\S]*?<\/\1>/gi, "");
+  text = text.replace(/<br\s*\/?>/gi, "\n");
+  text = text.replace(/<\/(p|div|tr|li)>/gi, "\n");
+  text = text.replace(/<[^>]+>/g, "");
+  text = text.replace(/&nbsp;/g, " ").replace(/&amp;/g, "&");
+  text = text.replace(/&lt;/g, "<").replace(/&gt;/g, ">");
+  text = text.replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+  text = text.replace(/&\w+;/g, "");
+  text = text.replace(/[ \t]+/g, " ");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  return text.trim();
+}
+
+function extractSnippet(textBody, htmlBody, maxLen = SNIPPET_LENGTH) {
+  let body = (textBody || "").trim();
+  if (!body && htmlBody) body = stripHtml(htmlBody);
+  return body.length > maxLen ? body.slice(0, maxLen) + "..." : body;
+}
+
+function isAlreadyProcessed(midHash, suffix) {
+  const baseName = `${midHash}-${suffix}.json`;
+  return (
+    existsSync(join(INBOX_DIR, baseName)) ||
+    existsSync(join(INBOX_DIR, baseName + ".processed"))
+  );
+}
+
+// ── Cursor Management ───────────────────────────────────────────────────
+
+function cursorPath(account) {
+  return join(SOPHIE_AI_DIR, "state", "polling", `${account}-gmail-cursor.yaml`);
+}
+
+function readImapCursor(account) {
+  const path = cursorPath(account);
+  try {
+    const content = readFileSync(path, "utf-8");
+    const cursor = {};
+    for (const line of content.split("\n")) {
+      const m = line.match(/^(\w+):\s*"?([^"\n]*)"?/);
+      if (m) {
+        let val = m[2].trim();
+        if (val === "null" || val === "") val = null;
+        cursor[m[1]] = val;
+      }
+    }
+    return {
+      last_poll: cursor.last_poll || null,
+      last_message_id: cursor.last_message_id || null,
+      messages_processed: parseInt(cursor.messages_processed || "0", 10) || 0,
+    };
+  } catch {
+    return { last_poll: null, last_message_id: null, messages_processed: 0 };
+  }
+}
+
+function writeImapCursor(account, lastPoll, lastMessageId, messagesProcessed) {
+  const path = cursorPath(account);
+  mkdirSync(dirname(path), { recursive: true });
+  const lp = lastPoll ? `"${lastPoll}"` : "null";
+  const lm = lastMessageId ? `"${lastMessageId}"` : "null";
+  writeFileSync(
+    path,
+    `last_poll: ${lp}\nlast_message_id: ${lm}\nmessages_processed: ${messagesProcessed}\n`,
+  );
+}
+
+// ── Attachment Extraction ───────────────────────────────────────────────
+
+function saveAttachments(parsed, midHash) {
+  const attachments = [];
+  if (!parsed.attachments || parsed.attachments.length === 0) return attachments;
+
+  mkdirSync(ATTACHMENTS_DIR, { recursive: true });
+  for (const att of parsed.attachments) {
+    if (!att.filename) continue;
+    // Skip tiny inline images (likely signatures)
+    if ((att.contentType || "").startsWith("image/") && att.content && att.content.length < 1024) continue;
+
+    const safeName = att.filename.replace(/[^a-zA-Z0-9._-]/g, "_");
+    const localName = `${midHash}-${safeName}`;
+    const localPath = join(ATTACHMENTS_DIR, localName);
+    try {
+      writeFileSync(localPath, att.content);
+      attachments.push({
+        name: att.filename,
+        mimetype: att.contentType || "unknown",
+        size: att.content.length,
+        local_path: localPath,
+      });
+    } catch {
+      // Non-fatal
+    }
+  }
+  return attachments;
+}
+
+// ── Core IMAP Poll ──────────────────────────────────────────────────────
+
+/**
+ * Poll a Gmail inbox via IMAP and write new messages as JSON to the inbox dir.
+ *
+ * @param {object} opts
+ * @param {string} opts.email       — Email address (e.g. sophie@adaptic.ai)
+ * @param {string} opts.password    — Gmail app password
+ * @param {string} opts.account     — Account slug for cursor/file naming (e.g. "sophie" or "mehran")
+ * @param {string} opts.fileSuffix  — JSON file suffix (e.g. "email" or "mehran-email")
+ * @param {string} opts.eventType   — Event type tag (e.g. "sophie_email" or "mehran_email")
+ * @param {string} opts.logPrefix   — Console log prefix (e.g. "[sophie-gmail]")
+ * @returns {Promise<{newCount: number, errors: string[]}>}
+ */
+export async function pollImapInbox({
+  email: emailAddr,
+  password,
+  account,
+  fileSuffix,
+  eventType,
+  logPrefix,
+}) {
+  const errors = [];
+
+  if (!password) {
+    console.log(`${logPrefix} App password not set, skipping`);
+    return { newCount: 0, errors };
+  }
+
+  mkdirSync(INBOX_DIR, { recursive: true });
+
+  const cursor = readImapCursor(account);
+  const now = new Date().toISOString();
+  let newCount = 0;
+  let skipped = 0;
+  let lastMid = cursor.last_message_id;
+  let totalProcessed = cursor.messages_processed;
+
+  const client = new ImapFlow({
+    host: "imap.gmail.com",
+    port: 993,
+    secure: true,
+    auth: { user: emailAddr, pass: password },
+    logger: false, // Suppress imapflow debug logs
+    socketTimeout: IMAP_TIMEOUT_MS,
+    greetTimeout: IMAP_TIMEOUT_MS,
+  });
+
+  try {
+    await client.connect();
+    const lock = await client.getMailboxLock("INBOX");
+
+    try {
+      // Search for unseen messages
+      const unseen = await client.search({ seen: false });
+      if (!unseen || unseen.length === 0) {
+        console.log(`${logPrefix} No unseen messages`);
+        writeImapCursor(account, now, lastMid, totalProcessed);
+        lock.release();
+        await client.logout();
+        return { newCount: 0, errors };
+      }
+
+      console.log(`${logPrefix} Found ${unseen.length} unseen messages`);
+
+      // Rate limit: process most recent N
+      const toFetch = unseen.length > MAX_EMAILS_PER_CYCLE
+        ? unseen.slice(-MAX_EMAILS_PER_CYCLE)
+        : unseen;
+
+      if (unseen.length > MAX_EMAILS_PER_CYCLE) {
+        console.log(`${logPrefix} Rate limited to last ${MAX_EMAILS_PER_CYCLE}`);
+      }
+
+      // Fetch messages using PEEK (don't mark as read)
+      for (const seq of toFetch) {
+        try {
+          const msg = await client.fetchOne(seq, {
+            envelope: true,
+            bodyStructure: true,
+            source: true, // Full raw source for body/attachment extraction
+          }, { uid: false });
+
+          if (!msg || !msg.envelope) continue;
+
+          const env = msg.envelope;
+          const fromAddr = env.from?.[0]?.address || "";
+          const fromName = env.from?.[0]?.name || fromAddr;
+          const subject = env.subject || "";
+          const dateStr = env.date ? env.date.toISOString() : "";
+          const messageId = env.messageId || "";
+
+          const midHash = messageIdHash(messageId, fromAddr, subject, dateStr);
+
+          if (isAlreadyProcessed(midHash, fileSuffix)) {
+            skipped++;
+            continue;
+          }
+
+          // Parse the full message for body and attachments
+          let snippet = "";
+          let emailAttachments = [];
+          if (msg.source) {
+            try {
+              const { simpleParser } = await import("mailparser");
+              const parsed = await simpleParser(msg.source);
+              snippet = extractSnippet(parsed.text, parsed.html);
+              emailAttachments = saveAttachments(parsed, midHash);
+            } catch {
+              // Fall back to envelope-only data
+              snippet = "";
+            }
+          }
+
+          const record = {
+            received_at: now,
+            event_type: eventType,
+            source: `${account}_gmail_poll`,
+            email: {
+              from: `${fromName} <${fromAddr}>`,
+              from_name: fromName,
+              from_addr: fromAddr,
+              to: emailAddr,
+              subject,
+              date: dateStr,
+              message_id: messageId,
+              snippet,
+            },
+            attachments: emailAttachments.length > 0 ? emailAttachments : null,
+            triage_hint: null,
+          };
+
+          const filepath = join(INBOX_DIR, `${midHash}-${fileSuffix}.json`);
+          writeFileSync(filepath, JSON.stringify(record, null, 2));
+
+          newCount++;
+          lastMid = messageId;
+          console.log(`${logPrefix} New: ${fromName} <${fromAddr}> — ${subject.slice(0, 60)}`);
+        } catch (err) {
+          errors.push(`${logPrefix} Message fetch error: ${err.message}`);
+        }
+      }
+    } finally {
+      lock.release();
+    }
+
+    totalProcessed += newCount;
+    writeImapCursor(account, now, lastMid, totalProcessed);
+    await client.logout();
+
+    console.log(
+      `${logPrefix} Poll complete: ${newCount} new, ${skipped} skipped, ${unseen.length} unseen total`,
+    );
+  } catch (err) {
+    errors.push(`${logPrefix} IMAP error: ${err.message}`);
+    console.error(`${logPrefix} IMAP error: ${err.message}`);
+    writeImapCursor(account, now, lastMid, totalProcessed);
+    try { await client.logout(); } catch { /* already disconnected */ }
+  }
+
+  return { newCount, errors };
+}

package/scripts/poller/index.mjs

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+// Sophie Poller — Lightweight event detection daemon
+// Runs every 60 seconds via macOS launchd
+// Polls Slack, Gmail, Calendar for new items
+// Writes to state/inbox/ for Sophie to process
+
+import { pollSlack } from "./slack-poller.mjs";
+import { pollGmail } from "./gmail-poller.mjs";
+import { pollCalendar } from "./calendar-poller.mjs";
+import { pollMehranGmail } from "./mehran-gmail-poller.mjs";
+import { triggerSophie } from "./trigger.mjs";
+import { isPriorityItem, SOPHIE_AI_DIR } from "./utils.mjs";
+import { appendFileSync, mkdirSync } from "fs";
+import { join } from "path";
+
+async function main() {
+  const timestamp = new Date().toISOString();
+  const logDir = join(SOPHIE_AI_DIR, "logs", "polling");
+  mkdirSync(logDir, { recursive: true });
+  const logFile = join(logDir, `${timestamp.split("T")[0]}-poller.jsonl`);
+
+  let totalItems = 0;
+  let totalErrors = 0;
+  let priorityTriggered = false;
+
+  const services = [
+    { name: "slack", fn: pollSlack },
+    { name: "gmail", fn: pollGmail },
+    { name: "calendar", fn: pollCalendar },
+    { name: "mehran-gmail", fn: pollMehranGmail },
+  ];
+
+  for (const svc of services) {
+    try {
+      const result = await svc.fn();
+      totalItems += result.items.length;
+      totalErrors += result.errors.length;
+
+      for (const item of result.items) {
+        if (isPriorityItem(item) && !priorityTriggered) {
+          triggerSophie(item);
+          priorityTriggered = true;
+        }
+      }
+
+      if (result.errors.length > 0) {
+        console.error(`[${svc.name}] Errors:`, result.errors);
+      }
+    } catch (err) {
+      totalErrors++;
+      console.error(`[${svc.name}] Poll failed:`, err.message);
+    }
+  }
+
+  appendFileSync(
+    logFile,
+    JSON.stringify({
+      timestamp,
+      items_found: totalItems,
+      errors: totalErrors,
+      priority_triggered: priorityTriggered,
+    }) + "\n",
+  );
+
+  if (totalItems > 0) {
+    console.log(`[poller] Found ${totalItems} new items`);
+  }
+}
+
+main().catch((err) => {
+  console.error("[poller] Fatal error:", err);
+  process.exit(1);
+});