@chainlesschain/personal-data-hub 0.1.0 → 0.2.1

package/lib/adapters/email-imap/email-adapter.js

@@ -28,9 +28,13 @@ const {
   ImapConnectionFailedError,
 } = require("./imap-session");
 const { parseRawEmail } = require("./email-parser");
+const { classifyEmail, CATEGORIES } = require("./classifier");
+const { extractFields } = require("./templates");
+const { extractPdfText, passwordsFromHints } = require("./pdf-extractor");
+const { extractTransactions } = require("./transactions");
 
 const NAME = "email-imap";
-const VERSION = "0.2.0"; // bumped for Phase 5.2 body parsing
+const VERSION = "0.6.0"; // Phase 5.7 — retry-with-backoff + progress streaming
 
 class EmailAdapter {
   constructor(opts) {
@@ -64,9 +68,78 @@ class EmailAdapter {
       ? opts.maxBodyChars
       : 8000;
 
+    // Phase 5.3: classifier configuration.
+    // - opts.llm: optional LLMClient for Layer 2 + Phase-5.4 `other`-template
+    //   summarization; absent → Layer 1 + regex-only extractors.
+    // - opts.classifier: custom orchestrator (override for tests).
+    // - opts.minLayer1Confidence: short-circuit threshold (default 0.85).
+    // - opts.disableClassification: skip both layers entirely.
+    this._llm = opts.llm && typeof opts.llm.chat === "function" ? opts.llm : null;
+    this._classifier = typeof opts.classifier === "function" ? opts.classifier : classifyEmail;
+    this._minLayer1Confidence = Number.isFinite(opts.minLayer1Confidence)
+      ? opts.minLayer1Confidence
+      : 0.85;
+    this._disableClassification = !!opts.disableClassification;
+
+    // Phase 5.4: template field extractor (regex-first, LLM-optional).
+    // - opts.extractor: custom dispatcher (test seam).
+    // - opts.disableExtraction: skip the per-email field-extraction call
+    //   (e.g. when the registry only needs envelope+classification).
+    this._extractor = typeof opts.extractor === "function" ? opts.extractor : extractFields;
+    this._disableExtraction = !!opts.disableExtraction;
+
+    // Phase 5.5: PDF attachment decryption + transactions extraction.
+    // See pdf-extractor.js + transactions.js. Test seam: opts.pdfExtractor.
+    this._pdfExtractor = typeof opts.pdfExtractor === "function" ? opts.pdfExtractor : extractPdfText;
+    this._transactionsExtractor = typeof opts.transactionsExtractor === "function"
+      ? opts.transactionsExtractor
+      : extractTransactions;
+    const hintsList = passwordsFromHints(opts.pdfPasswordHints || {});
+    const userList = Array.isArray(opts.pdfPasswords)
+      ? opts.pdfPasswords.filter((p) => typeof p === "string")
+      : [];
+    this._pdfPasswords = [...userList, ...hintsList].filter((v, i, arr) => arr.indexOf(v) === i);
+    this._disablePdfExtraction = !!opts.disablePdfExtraction;
+
+    // Phase 5.7: connection retry + progress streaming.
+    // - opts.maxConnectRetries (default 3): total attempts including first.
+    //   Set to 1 to disable retry. Retries fire ONLY on transient errors
+    //   (ECONNRESET / ETIMEDOUT / EPIPE / socket disconnects); AUTH_FAILED
+    //   and MAILBOX_NOT_FOUND short-circuit.
+    // - opts.retryBaseDelayMs (default 200): exponential backoff base.
+    //   Actual delays: 200ms → 600ms → 1800ms for 3 attempts.
+    // - opts.onProgress (callback): receives {phase, ...payload} events
+    //   throughout sync(). Registry forwards via onSyncEvent so the WS/IPC
+    //   layer can stream to UI. Phases:
+    //     "connecting"   {attempt}
+    //     "connected"
+    //     "mailbox-opened" {mailbox, exists}
+    //     "fetching"     {mailbox, current, total}
+    //     "decrypting-pdf" {filename}
+    //     "done"         {emitted, durationMs}
+    //     "error"        {phase, message, retriable?}
+    this._maxConnectRetries = Number.isFinite(opts.maxConnectRetries) && opts.maxConnectRetries > 0
+      ? opts.maxConnectRetries
+      : 3;
+    this._retryBaseDelayMs = Number.isFinite(opts.retryBaseDelayMs) && opts.retryBaseDelayMs > 0
+      ? opts.retryBaseDelayMs
+      : 200;
+    this._onProgress = typeof opts.onProgress === "function" ? opts.onProgress : null;
+
     this.name = NAME;
     this.version = VERSION;
-    this.capabilities = ["sync:imap", "auth:authcode", "parse:mime-body", "parse:attachment-metadata"];
+    this.capabilities = [
+      "sync:imap",
+      "auth:authcode",
+      "parse:mime-body",
+      "parse:attachment-metadata",
+      "classify:layer1-rules",
+      ...(this._llm ? ["classify:layer2-llm"] : []),
+      "extract:6-templates",
+      ...(this._disablePdfExtraction ? [] : ["decrypt:pdf-bills"]),
+      "sync:retry-backoff",
+      "sync:progress-stream",
+    ];
     this.rateLimits = { perMinute: 60 };
     this.dataDisclosure = {
       fields: [
@@ -74,6 +147,12 @@ class EmailAdapter {
         "email:flags + uid + internalDate",
         "email:body (text + html, capped to ~8k chars)",
         "email:attachment-metadata (filename, contentType, size, sha256; no file bytes saved in v0)",
+        "classification:layer-1-rule-or-layer-2-llm-category (bill_bank/order/travel/etc.)",
+        ...(this._disablePdfExtraction
+          ? []
+          : [
+              "bill-transactions:date+description+amount+balance (extracted from decrypted PDF attachments; PDF bytes themselves never persist)",
+            ]),
       ],
       sensitivity: "high",
       legalGate: false,
@@ -114,17 +193,46 @@ class EmailAdapter {
     const watermark = typeof opts.sinceWatermark === "string" ? opts.sinceWatermark : "";
     const { uidValidity: prevUv, lastUid: prevLastUid } = parseWatermark(watermark);
 
+    // Phase 5.7: per-sync progress hook is the union of constructor + opts.
+    // Callers (registry / tests) can pass a fresh callback per sync without
+    // mutating the adapter instance.
+    const syncOnProgress = typeof opts.onProgress === "function"
+      ? opts.onProgress
+      : this._onProgress;
+    const emitProgress = (phase, payload = {}) => {
+      if (!syncOnProgress) return;
+      try {
+        syncOnProgress({ phase, adapter: NAME, ...payload });
+      } catch (_e) {
+        // Listener errors must NOT abort the sync.
+      }
+    };
+
+    const syncStart = Date.now();
     const session = this._sessionFactory(this._sessionConfig());
+    let totalEmitted = 0;
     try {
-      await session.connect();
+      // Phase 5.7: connect with retry on transient errors.
+      await this._connectWithRetry(session, emitProgress);
 
       for (const folder of folders) {
         const mb = await session.openMailbox(folder);
+        emitProgress("mailbox-opened", {
+          mailbox: folder,
+          exists: mb.exists,
+          uidValidity: mb.uidValidity,
+        });
         const uvChanged = prevUv !== null && String(prevUv) !== String(mb.uidValidity);
         const since = uvChanged ? 0 : prevLastUid;
 
         let emitted = 0;
         for await (const env of session.fetchFullSince(since)) {
+          emitProgress("fetching", {
+            mailbox: folder,
+            current: emitted + 1,
+            total: mb.exists,
+            uid: env.uid,
+          });
           // Parse the body in the adapter (not the session) so the
           // session stays a thin IMAP wrapper. Parse failures degrade
           // gracefully — emit the raw event without parsedBody so the
@@ -133,25 +241,132 @@ class EmailAdapter {
           let parsedBody = null;
           try {
             if (env.source && env.source.length > 0) {
-              parsedBody = await this._parser(env.source);
+              // Phase 5.5: ask parser to keep attachment buffers when
+              // we may need to decrypt PDFs. Buffers are stripped from
+              // the emitted RawEvent (in _envelopeToRawEvent) so the
+              // vault doesn't archive megabytes of PDF bytes.
+              parsedBody = await this._parser(env.source, {
+                keepAttachmentBuffers: !this._disablePdfExtraction,
+              });
             }
           } catch (parseErr) {
-            // Phase 5.3 classifier rules can still fire on envelope-only
+            // Layer 1 classifier rules can still fire on envelope-only
             // facts; we just lose body text + attachments for this email.
             parsedBody = {
               parseError: parseErr && parseErr.message ? parseErr.message : String(parseErr),
             };
           }
-          yield this._envelopeToRawEvent(env, folder, parsedBody);
+
+          // Phase 5.3: classify. Layer 1 runs synchronously on
+          // (from, subject, headers, attachment hints). If under the
+          // confidence threshold AND we have an LLM, Layer 2 fires.
+          // Classifier errors degrade to OTHER (never abort sync).
+          let classification = null;
+          if (!this._disableClassification) {
+            try {
+              classification = await this._classifier(
+                this._classifierInput(env, parsedBody),
+                {
+                  llm: this._llm,
+                  minLayer1Confidence: this._minLayer1Confidence,
+                }
+              );
+            } catch (err) {
+              classification = {
+                category: CATEGORIES.OTHER,
+                confidence: 0,
+                layer: "error",
+                error: err && err.message ? err.message : String(err),
+              };
+            }
+          }
+
+          // Phase 5.4: per-category template extraction.
+          // Dispatcher routes via classification.category, so a missing
+          // classification (disableClassification=true) maps to "other".
+          let extraction = null;
+          if (!this._disableExtraction) {
+            try {
+              extraction = await this._extractor(
+                this._classifierInput(env, parsedBody),
+                classification || { category: CATEGORIES.OTHER },
+                { llm: this._llm }
+              );
+            } catch (err) {
+              extraction = {
+                template: "other",
+                fields: {},
+                confidence: 0,
+                warnings: [`extractor threw: ${err && err.message ? err.message : err}`],
+              };
+            }
+          }
+
+          // Phase 5.5: PDF attachment decryption + transactions extraction.
+          // Runs only when the email was classified as a bill / travel AND
+          // has at least one PDF attachment whose buffer is available.
+          // Errors captured per-attachment, never thrown.
+          if (
+            !this._disablePdfExtraction
+            && extraction
+            && (extraction.template === "bill" || extraction.template === "travel")
+            && parsedBody
+            && Array.isArray(parsedBody.attachments)
+            && parsedBody.attachments.some((a) => isPdfAttachment(a))
+          ) {
+            await this._runPdfExtraction(parsedBody, extraction);
+          }
+
+          yield this._envelopeToRawEvent(env, folder, parsedBody, classification, extraction);
           emitted += 1;
+          totalEmitted += 1;
           if (emitted >= maxPerFolder) break;
         }
       }
+      emitProgress("done", { emitted: totalEmitted, durationMs: Date.now() - syncStart });
     } finally {
       try { await session.close(); } catch (_e) {}
     }
   }
 
+  /**
+   * Phase 5.7: connect with retry on transient errors. Auth failures
+   * (AUTH_FAILED) and mailbox-not-found (MAILBOX_NOT_FOUND) bypass retry —
+   * those are user errors that won't fix themselves. Network blips
+   * (ECONNRESET / ETIMEDOUT / EPIPE / socket errors / generic
+   * CONNECTION_FAILED) get up to `_maxConnectRetries` attempts with
+   * exponential backoff.
+   */
+  async _connectWithRetry(session, emitProgress) {
+    const maxAttempts = this._maxConnectRetries;
+    let lastErr = null;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+      emitProgress("connecting", { attempt, maxAttempts });
+      try {
+        await session.connect();
+        emitProgress("connected", { attempt });
+        return;
+      } catch (err) {
+        lastErr = err;
+        const transient = isTransientImapError(err);
+        emitProgress("error", {
+          failingPhase: "connecting",
+          attempt,
+          retriable: transient && attempt < maxAttempts,
+          code: err && err.code,
+          message: err && err.message ? err.message : String(err),
+        });
+        if (!transient || attempt >= maxAttempts) {
+          throw err;
+        }
+        // Exponential backoff: base * 3^(attempt-1)
+        const delay = this._retryBaseDelayMs * Math.pow(3, attempt - 1);
+        await sleep(delay);
+      }
+    }
+    throw lastErr;
+  }
+
   normalize(raw) {
     if (!raw || typeof raw !== "object" || !raw.payload) {
       throw new Error("EmailAdapter.normalize: missing raw or raw.payload");
@@ -251,9 +466,54 @@ class EmailAdapter {
               indicatorHeaders: pickIndicatorHeaders(parsedBody.headers),
             }
           : {}),
+        // Phase 5.3: per-email category + which layer / rule decided it.
+        // Phase 5.4 template extractors dispatched on `.classified`.
+        ...(env.classification
+          ? {
+              classified: env.classification.category,
+              classification: {
+                category: env.classification.category,
+                confidence: env.classification.confidence,
+                layer: env.classification.layer,
+                ...(env.classification.ruleName ? { ruleName: env.classification.ruleName } : {}),
+                ...(env.classification.reason ? { reason: env.classification.reason } : {}),
+              },
+            }
+          : {}),
+        // Phase 5.4: structured fields from the per-category template.
+        // Stored at top of extra so analysis prompts + KG ingestors can
+        // see them without spelunking through `extraction.fields`.
+        ...(env.extraction && env.extraction.fields
+          ? {
+              fields: env.extraction.fields,
+              extractionTemplate: env.extraction.template,
+              extractionConfidence: env.extraction.confidence,
+              ...(env.extraction.warnings && env.extraction.warnings.length > 0
+                ? { extractionWarnings: env.extraction.warnings }
+                : {}),
+              // Phase 5.5: per-attachment decrypt+extract summary so the
+              // UI can flag "could not unlock this bill" + transactions
+              // count. Actual transactions list lives at fields.transactions.
+              ...(env.extraction.pdfExtraction
+                ? { pdfExtraction: env.extraction.pdfExtraction }
+                : {}),
+            }
+          : {}),
       },
     };
 
+    // Phase 5.4 compliance redaction: emails containing verification
+    // codes (OTP / 2FA) must NEVER persist their body in vault — even
+    // an "expired" OTP is sensitive evidence of session activity.
+    // Adapter_Email_IMAP.md §9.2 mandates "verificationCodePresent =
+    // store metadata only".
+    if (env.extraction
+        && env.extraction.template === "register"
+        && env.extraction.fields
+        && env.extraction.fields.verificationCodePresent) {
+      event.content.text = "(redacted: verification code email)";
+    }
+
     return { events: [event], persons, places: [], items: [], topics: [] };
   }
 
@@ -267,7 +527,7 @@ class EmailAdapter {
     };
   }
 
-  _envelopeToRawEvent(env, folder, parsedBody) {
+  _envelopeToRawEvent(env, folder, parsedBody, classification, extraction) {
     const originalId = env.messageId && env.messageId.length > 0
       ? env.messageId
       : `mid-fallback:${this.account.email}:${folder}:${env.uid}`;
@@ -280,6 +540,10 @@ class EmailAdapter {
     // re-derive). The source is recoverable by re-syncing if absolutely
     // needed.
     const { source: _src, ...envNoSource } = env;
+    // Phase 5.5: also strip attachment buffers from parsedBody. Buffers
+    // are loaded for PDF decryption then discarded — vault keeps only
+    // metadata (filename / contentType / size / sha256).
+    const safeBody = parsedBody ? stripAttachmentBuffers(parsedBody) : null;
     return {
       adapter: NAME,
       originalId,
@@ -287,11 +551,90 @@ class EmailAdapter {
       payload: {
         ...envNoSource,
         folder,
-        ...(parsedBody ? { parsedBody } : {}),
+        ...(safeBody ? { parsedBody: safeBody } : {}),
+        ...(classification ? { classification } : {}),
+        ...(extraction ? { extraction } : {}),
       },
     };
   }
 
+  /**
+   * Phase 5.5 helper: for each PDF attachment with a buffer, try to
+   * decrypt + extract text + parse transactions. Merges results into
+   * `extraction.fields.transactions` and stamps `pdfExtraction` metadata
+   * on the extraction so UI can surface failures.
+   *
+   * Side effects: mutates `extraction.fields` + adds `extraction.pdfExtraction`.
+   * Errors captured, never thrown — preserves the "sync never aborts on
+   * a single bad email" invariant.
+   */
+  async _runPdfExtraction(parsedBody, extraction) {
+    const pdfAtts = parsedBody.attachments.filter((a) => isPdfAttachment(a));
+    const results = [];
+    const allTxns = [];
+
+    for (const a of pdfAtts) {
+      if (!Buffer.isBuffer(a.buffer)) {
+        results.push({
+          filename: a.filename,
+          decrypted: false,
+          attempted: 0,
+          error: "no buffer (parser keepAttachmentBuffers=false?)",
+        });
+        continue;
+      }
+      try {
+        const r = await this._pdfExtractor(a.buffer, { passwords: this._pdfPasswords });
+        const summary = {
+          filename: a.filename,
+          decrypted: r.decrypted,
+          attempted: r.attempted,
+          wasEncrypted: r.wasEncrypted,
+          pageCount: r.pageCount,
+          ...(r.password !== undefined ? { passwordUsed: "***" } : {}), // never persist the real password
+          ...(r.error ? { error: r.error } : {}),
+        };
+        if (r.decrypted && typeof r.text === "string" && r.text.length > 0) {
+          const txns = this._transactionsExtractor(r.text);
+          if (Array.isArray(txns) && txns.length > 0) {
+            for (const t of txns) {
+              t.attachmentSha256 = a.sha256;
+              allTxns.push(t);
+            }
+            summary.transactionsExtracted = txns.length;
+          } else {
+            summary.transactionsExtracted = 0;
+          }
+        }
+        results.push(summary);
+      } catch (err) {
+        results.push({
+          filename: a.filename,
+          decrypted: false,
+          attempted: 0,
+          error: err && err.message ? err.message : String(err),
+        });
+      }
+    }
+
+    if (allTxns.length > 0) {
+      extraction.fields = extraction.fields || {};
+      extraction.fields.transactions = allTxns;
+    }
+    extraction.pdfExtraction = results;
+  }
+
+  _classifierInput(env, parsedBody) {
+    return {
+      from: env.from,
+      subject: env.subject,
+      attachments: parsedBody && Array.isArray(parsedBody.attachments) ? parsedBody.attachments : [],
+      textBody: (parsedBody && parsedBody.textBody) || "",
+      htmlBody: (parsedBody && parsedBody.htmlBody) || "",
+      headers: parsedBody && parsedBody.headers ? parsedBody.headers : {},
+    };
+  }
+
   _source(originalId, internalDate) {
     return {
       adapter: NAME,
@@ -389,6 +732,74 @@ function pickIndicatorHeaders(headers) {
   return out;
 }
 
+/**
+ * Phase 5.7: decide if an IMAP error is worth retrying. Transient
+ * network blips (ECONNRESET / ETIMEDOUT / EPIPE / connect-failed / socket-
+ * disconnect / "connection lost") get a retry; auth failures / mailbox
+ * misconfig do NOT.
+ */
+function isTransientImapError(err) {
+  if (!err) return false;
+  if (err.code === "AUTH_FAILED" || err.code === "MAILBOX_NOT_FOUND") return false;
+  if (err.code === "CONNECTION_FAILED") return true;
+  // Node-level network error codes
+  const networkCodes = new Set([
+    "ECONNRESET",
+    "ETIMEDOUT",
+    "EPIPE",
+    "ECONNREFUSED",
+    "ENETUNREACH",
+    "EAI_AGAIN",
+    "ENOTFOUND", // DNS can be transient on flaky networks
+  ]);
+  if (err.code && networkCodes.has(err.code)) return true;
+  if (err.cause && err.cause.code && networkCodes.has(err.cause.code)) return true;
+  const msg = (err.message || "").toLowerCase();
+  if (
+    msg.includes("timed out")
+    || msg.includes("timeout")
+    || msg.includes("socket disconnect")
+    || msg.includes("connection lost")
+    || msg.includes("connection reset")
+    || msg.includes("write after end")
+  ) {
+    return true;
+  }
+  return false;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Phase 5.5: detect PDF attachments. Goes by contentType first, falls
+ * back to filename extension for senders that omit the MIME type.
+ */
+function isPdfAttachment(a) {
+  if (!a || typeof a !== "object") return false;
+  if (typeof a.contentType === "string" && a.contentType.toLowerCase().includes("pdf")) return true;
+  if (typeof a.filename === "string" && /\.pdf$/i.test(a.filename)) return true;
+  return false;
+}
+
+/**
+ * Phase 5.5: drop Buffer fields from each attachment before the parsed
+ * body lands in the emitted RawEvent. Vault row size + WS-gateway
+ * serialization cost would be dominated by attachment bytes otherwise.
+ */
+function stripAttachmentBuffers(parsedBody) {
+  if (!parsedBody || !Array.isArray(parsedBody.attachments)) return parsedBody;
+  return {
+    ...parsedBody,
+    attachments: parsedBody.attachments.map((a) => {
+      if (!a || a.buffer == null) return a;
+      const { buffer: _b, ...rest } = a;
+      return rest;
+    }),
+  };
+}
+
 module.exports = {
   EmailAdapter,
   parseWatermark,

package/lib/adapters/email-imap/index.js

@@ -9,6 +9,27 @@ const {
   ImapMailboxNotFoundError,
 } = require("./imap-session");
 const { parseRawEmail } = require("./email-parser");
+const {
+  CATEGORIES,
+  ALL_CATEGORIES,
+  LAYER1_RULES,
+  classifyLayer1,
+  classifyLayer2,
+  classifyEmail,
+  LAYER2_SYSTEM_PROMPT,
+} = require("./classifier");
+const {
+  extractFields,
+  CATEGORY_TO_EXTRACTOR,
+  extractBill,
+  extractOrder,
+  extractTravel,
+  extractGovernment,
+  extractRegister,
+  extractOther,
+} = require("./templates");
+const { extractPdfText, passwordsFromHints } = require("./pdf-extractor");
+const { extractTransactions } = require("./transactions");
 
 module.exports = {
   EmailAdapter,
@@ -23,4 +44,25 @@ module.exports = {
   ImapConnectionFailedError,
   ImapMailboxNotFoundError,
   parseRawEmail,
+  // Phase 5.3 classifier
+  EMAIL_CATEGORIES: CATEGORIES,
+  EMAIL_ALL_CATEGORIES: ALL_CATEGORIES,
+  EMAIL_LAYER1_RULES: LAYER1_RULES,
+  classifyEmailLayer1: classifyLayer1,
+  classifyEmailLayer2: classifyLayer2,
+  classifyEmail,
+  EMAIL_LAYER2_SYSTEM_PROMPT: LAYER2_SYSTEM_PROMPT,
+  // Phase 5.4 — 6 template extractors + dispatcher
+  extractEmailFields: extractFields,
+  EMAIL_CATEGORY_EXTRACTORS: CATEGORY_TO_EXTRACTOR,
+  extractBill,
+  extractOrder,
+  extractTravel,
+  extractGovernment,
+  extractRegister,
+  extractOther,
+  // Phase 5.5 — PDF decryption + transactions extraction
+  extractPdfText,
+  pdfPasswordsFromHints: passwordsFromHints,
+  extractTransactions,
 };