@chainlesschain/personal-data-hub 0.1.0

package/lib/adapters/email-imap/imap-session.js

@@ -0,0 +1,294 @@
+/**
+ * Thin async-iterator wrapper around imapflow.
+ *
+ * The hub never imports imapflow directly — this file is the single
+ * static require boundary. EmailAdapter takes an ImapSession instance
+ * (or any object with the same surface) so tests can inject a mock
+ * without imapflow even being installed.
+ *
+ * Errors are surfaced with normalized `code`:
+ *   AUTH_FAILED          login rejected
+ *   CONNECTION_FAILED    TCP / TLS failure
+ *   MAILBOX_NOT_FOUND    folder doesn't exist
+ */
+
+"use strict";
+
+class ImapAuthFailedError extends Error {
+  constructor(message, cause) {
+    super(message || "IMAP authentication failed");
+    this.name = "ImapAuthFailedError";
+    this.code = "AUTH_FAILED";
+    if (cause) this.cause = cause;
+  }
+}
+
+class ImapConnectionFailedError extends Error {
+  constructor(message, cause) {
+    super(message || "IMAP connection failed");
+    this.name = "ImapConnectionFailedError";
+    this.code = "CONNECTION_FAILED";
+    if (cause) this.cause = cause;
+  }
+}
+
+class ImapMailboxNotFoundError extends Error {
+  constructor(name) {
+    super(`Mailbox not found: ${name}`);
+    this.name = "ImapMailboxNotFoundError";
+    this.code = "MAILBOX_NOT_FOUND";
+    this.mailbox = name;
+  }
+}
+
+class ImapSession {
+  constructor(opts) {
+    if (!opts || typeof opts !== "object") {
+      throw new Error("ImapSession: opts required");
+    }
+    for (const f of ["host", "port", "user", "authCode"]) {
+      if (opts[f] === undefined || opts[f] === null || opts[f] === "") {
+        throw new Error(`ImapSession: opts.${f} required`);
+      }
+    }
+    this.host = String(opts.host);
+    this.port = Number(opts.port);
+    this.secure = opts.secure !== false;
+    this.user = String(opts.user);
+    this.authCode = String(opts.authCode);
+    this.connectTimeoutMs = Number.isFinite(opts.connectTimeoutMs)
+      ? opts.connectTimeoutMs
+      : 15000;
+    this._factory = typeof opts.imapFlowFactory === "function" ? opts.imapFlowFactory : null;
+    this._client = null;
+  }
+
+  async connect() {
+    let ImapFlowCtor;
+    if (this._factory) {
+      ImapFlowCtor = this._factory;
+    } else {
+      try {
+        const mod = require("imapflow");
+        ImapFlowCtor = mod.ImapFlow || mod.default || mod;
+      } catch (err) {
+        throw new ImapConnectionFailedError(
+          "imapflow is not installed. Run `npm install imapflow` in the workspace.",
+          err
+        );
+      }
+    }
+
+    const ctorOpts = {
+      host: this.host,
+      port: this.port,
+      secure: this.secure,
+      auth: { user: this.user, pass: this.authCode },
+      logger: false,
+    };
+    let client;
+    try {
+      // Real imapflow's ImapFlow is an ES class → must use `new`. Test
+      // injection sometimes passes an arrow factory which `new` rejects
+      // (TypeError: not a constructor). Try constructor first, fall
+      // through to plain call so both shapes work.
+      try {
+        client = new ImapFlowCtor(ctorOpts);
+      } catch (ctorErr) {
+        if (ctorErr instanceof TypeError) {
+          client = ImapFlowCtor(ctorOpts);
+        } else {
+          throw ctorErr;
+        }
+      }
+    } catch (err) {
+      throw new ImapConnectionFailedError(
+        `Failed to construct IMAP client: ${err && err.message ? err.message : err}`,
+        err
+      );
+    }
+
+    let connectPromise;
+    try {
+      connectPromise = client.connect();
+    } catch (err) {
+      throw new ImapConnectionFailedError(
+        `IMAP connect threw synchronously: ${err && err.message ? err.message : err}`,
+        err
+      );
+    }
+
+    let timer = null;
+    const timeoutPromise = new Promise((_, reject) => {
+      timer = setTimeout(() => {
+        reject(new ImapConnectionFailedError(
+          `IMAP connect timed out after ${this.connectTimeoutMs}ms`
+        ));
+      }, this.connectTimeoutMs);
+    });
+
+    try {
+      await Promise.race([connectPromise, timeoutPromise]);
+    } catch (err) {
+      try { await client.close(); } catch (_e) {}
+      const msg = (err && err.message ? err.message : String(err)).toLowerCase();
+      if (msg.includes("auth") || msg.includes("invalid credentials") || msg.includes("login") || msg.includes("rejected")) {
+        throw new ImapAuthFailedError(err && err.message, err);
+      }
+      if (err && err.code === "AUTH_FAILED") throw err;
+      if (err && err.code === "CONNECTION_FAILED") throw err;
+      throw new ImapConnectionFailedError(
+        `IMAP connect failed: ${err && err.message ? err.message : err}`,
+        err
+      );
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+
+    this._client = client;
+  }
+
+  _requireConnected() {
+    if (!this._client) {
+      throw new Error("ImapSession: not connected; call connect() first.");
+    }
+    return this._client;
+  }
+
+  async openMailbox(name) {
+    const c = this._requireConnected();
+    if (typeof name !== "string" || !name) {
+      throw new Error("openMailbox: name must be a non-empty string");
+    }
+    let info;
+    try {
+      info = await c.mailboxOpen(name);
+    } catch (err) {
+      const msg = (err && err.message ? err.message : "").toLowerCase();
+      if (msg.includes("doesn't exist") || msg.includes("nonexistent") || msg.includes("not found")) {
+        throw new ImapMailboxNotFoundError(name);
+      }
+      throw err;
+    }
+    return {
+      uidValidity: info.uidValidity,
+      uidNext: info.uidNext,
+      exists: info.exists,
+    };
+  }
+
+  async listMailboxes() {
+    const c = this._requireConnected();
+    const items = await c.list();
+    if (!Array.isArray(items)) return [];
+    return items.map((m) => ({
+      name: m.name,
+      path: m.path,
+      specialUse: m.specialUse || null,
+      flags: Array.isArray(m.flags) ? m.flags : [],
+    }));
+  }
+
+  async *fetchEnvelopesSince(sinceUid = 0) {
+    const c = this._requireConnected();
+    const baseUid = Number.isFinite(sinceUid) && sinceUid > 0 ? sinceUid : 0;
+    const range = `${baseUid + 1}:*`;
+    const fields = { envelope: true, internalDate: true, flags: true, size: true, uid: true };
+    const iter = c.fetch(range, fields, { uid: true });
+    for await (const msg of iter) {
+      yield this._toEnvelopeRow(msg);
+    }
+  }
+
+  /**
+   * Like fetchEnvelopesSince but also pulls the full RFC822 source of
+   * each message (`source: true`). Phase 5.2 — the EmailAdapter feeds
+   * these into the mailparser-based email-parser to extract body text,
+   * HTML, attachments metadata, etc.
+   *
+   * Memory note: source bytes pile up in memory until each generator
+   * consumer awaits the next iteration. For huge mailboxes the registry's
+   * batchSize (default 100) acts as the natural back-pressure — every
+   * `batchSize` messages get committed to vault before the next batch
+   * pulls. Phase 5.5 PDF-decryption work will switch to per-attachment
+   * download for emails > N MB.
+   *
+   * Yields `{ ...envelopeRow, source: Buffer }`.
+   *
+   * @param {number} sinceUid
+   * @returns {AsyncGenerator}
+   */
+  async *fetchFullSince(sinceUid = 0) {
+    const c = this._requireConnected();
+    const baseUid = Number.isFinite(sinceUid) && sinceUid > 0 ? sinceUid : 0;
+    const range = `${baseUid + 1}:*`;
+    const fields = {
+      envelope: true,
+      internalDate: true,
+      flags: true,
+      size: true,
+      uid: true,
+      source: true, // raw RFC822 bytes
+    };
+    const iter = c.fetch(range, fields, { uid: true });
+    for await (const msg of iter) {
+      const row = this._toEnvelopeRow(msg);
+      // imapflow returns source as a Buffer; defensively coerce.
+      row.source = Buffer.isBuffer(msg.source)
+        ? msg.source
+        : msg.source
+          ? Buffer.from(msg.source)
+          : Buffer.alloc(0);
+      yield row;
+    }
+  }
+
+  async close() {
+    if (!this._client) return;
+    try {
+      await this._client.logout();
+    } catch (_err) {}
+    try {
+      await this._client.close();
+    } catch (_err) {}
+    this._client = null;
+  }
+
+  _toEnvelopeRow(msg) {
+    const env = msg.envelope || {};
+    // imapflow returns flags as a Set; older shapes use Array. Cover both.
+    let flags = [];
+    if (msg.flags) {
+      if (msg.flags instanceof Set || Array.isArray(msg.flags) || typeof msg.flags[Symbol.iterator] === "function") {
+        flags = Array.from(msg.flags);
+      }
+    }
+    return {
+      uid: msg.uid,
+      internalDate: msg.internalDate instanceof Date ? msg.internalDate : new Date(msg.internalDate || 0),
+      flags,
+      messageId: typeof env.messageId === "string" ? env.messageId : "",
+      subject: typeof env.subject === "string" ? env.subject : "",
+      from: this._addrs(env.from),
+      to: this._addrs(env.to),
+      cc: this._addrs(env.cc),
+      date: env.date instanceof Date ? env.date : env.date ? new Date(env.date) : null,
+      size: typeof msg.size === "number" ? msg.size : 0,
+    };
+  }
+
+  _addrs(list) {
+    if (!Array.isArray(list)) return [];
+    return list.map((a) => ({
+      name: a && a.name ? String(a.name) : undefined,
+      address: a && a.address ? String(a.address) : "",
+    }));
+  }
+}
+
+module.exports = {
+  ImapSession,
+  ImapAuthFailedError,
+  ImapConnectionFailedError,
+  ImapMailboxNotFoundError,
+};

package/lib/adapters/email-imap/index.js

@@ -0,0 +1,26 @@
+"use strict";
+
+const { EmailAdapter, parseWatermark, formatWatermark, NAME, VERSION } = require("./email-adapter");
+const { PROVIDERS, resolveProvider } = require("./providers");
+const {
+  ImapSession,
+  ImapAuthFailedError,
+  ImapConnectionFailedError,
+  ImapMailboxNotFoundError,
+} = require("./imap-session");
+const { parseRawEmail } = require("./email-parser");
+
+module.exports = {
+  EmailAdapter,
+  EMAIL_ADAPTER_NAME: NAME,
+  EMAIL_ADAPTER_VERSION: VERSION,
+  parseWatermark,
+  formatWatermark,
+  EMAIL_PROVIDERS: PROVIDERS,
+  resolveEmailProvider: resolveProvider,
+  ImapSession,
+  ImapAuthFailedError,
+  ImapConnectionFailedError,
+  ImapMailboxNotFoundError,
+  parseRawEmail,
+};

package/lib/adapters/email-imap/providers.js

@@ -0,0 +1,111 @@
+/**
+ * Email provider presets — mainland-China-first.
+ *
+ * Each entry tells the adapter where to connect + which folders are
+ * worth syncing by default. Users can override host/port/tls via the
+ * adapter constructor (provider="custom").
+ *
+ * Authentication is consistently the per-provider "authorization code"
+ * pattern (per design doc §3 OQ-1). The user goes to their email
+ * web console, enables IMAP/SMTP, copies the auth code, pastes it
+ * into the adapter config. The adapter never sees the user's actual
+ * login password.
+ */
+
+"use strict";
+
+const PROVIDERS = Object.freeze({
+  qq: {
+    id: "qq",
+    displayName: "QQ 邮箱",
+    host: "imap.qq.com",
+    port: 993,
+    secure: true,
+    setupUrl: "https://mail.qq.com/cgi-bin/frame_html?sid=&r=&t=client",
+    defaultFolders: ["INBOX", "Sent Messages"],
+    authNote: "Use 授权码 (设置 → 账户 → 开启 IMAP/SMTP), NOT your QQ login password.",
+  },
+  "189": {
+    id: "189",
+    displayName: "189 邮箱",
+    host: "imap.189.cn",
+    port: 993,
+    secure: true,
+    setupUrl: "https://mail.189.cn/",
+    defaultFolders: ["INBOX", "已发送"],
+    authNote: "Use 授权码 (设置 → 第三方客户端授权码).",
+  },
+  "163": {
+    id: "163",
+    displayName: "网易邮箱 (163/126)",
+    host: "imap.163.com",
+    port: 993,
+    secure: true,
+    setupUrl: "https://mail.163.com/",
+    defaultFolders: ["INBOX", "已发送"],
+    authNote: "Use 授权码 (设置 → POP3/SMTP/IMAP).",
+  },
+  outlook: {
+    id: "outlook",
+    displayName: "Outlook / Hotmail",
+    host: "outlook.office365.com",
+    port: 993,
+    secure: true,
+    setupUrl: "https://outlook.live.com/mail/0/options/mail/forwarding",
+    defaultFolders: ["INBOX", "Sent"],
+    authNote: "App password (account.microsoft.com/security) — basic-auth deprecation pending; v1 will switch to OAuth2.",
+  },
+  gmail: {
+    id: "gmail",
+    displayName: "Gmail",
+    host: "imap.gmail.com",
+    port: 993,
+    secure: true,
+    setupUrl: "https://myaccount.google.com/apppasswords",
+    defaultFolders: ["INBOX", "[Gmail]/Sent Mail"],
+    authNote: "App password (myaccount.google.com/apppasswords). OAuth2 in v2.",
+  },
+});
+
+function resolveProvider(account) {
+  if (!account || typeof account !== "object") {
+    throw new Error("resolveProvider: account required");
+  }
+  const id = account.provider;
+  if (id === "custom") {
+    if (typeof account.host !== "string" || !account.host) {
+      throw new Error("resolveProvider: custom provider requires host");
+    }
+    return {
+      host: account.host,
+      port: Number.isInteger(account.port) ? account.port : 993,
+      secure: account.secure !== false,
+      folders: Array.isArray(account.folders) && account.folders.length > 0
+        ? account.folders
+        : ["INBOX"],
+      displayName: account.displayName || account.host,
+      providerId: "custom",
+    };
+  }
+  const preset = PROVIDERS[id];
+  if (!preset) {
+    throw new Error(
+      `resolveProvider: unknown provider "${id}". Known: ${Object.keys(PROVIDERS).join(", ")}, or use "custom".`
+    );
+  }
+  return {
+    host: account.host || preset.host,
+    port: Number.isInteger(account.port) ? account.port : preset.port,
+    secure: typeof account.secure === "boolean" ? account.secure : preset.secure,
+    folders: Array.isArray(account.folders) && account.folders.length > 0
+      ? account.folders
+      : preset.defaultFolders,
+    displayName: account.displayName || preset.displayName,
+    providerId: preset.id,
+  };
+}
+
+module.exports = {
+  PROVIDERS,
+  resolveProvider,
+};

package/lib/analysis.js

@@ -0,0 +1,226 @@
+/**
+ * AnalysisEngine — natural-language Q&A skeleton for Personal Data Hub.
+ *
+ * Mirrors §8 of docs/design/Personal_Data_Hub_Architecture.md. The flow:
+ *
+ *   ask(question) →
+ *     1. parseQuery(question)        — time window + filters + intent
+ *     2. gatherFacts(parsed)         — vault.queryEvents with filters
+ *        + optional ragRetriever(question)  for additional context
+ *     3. buildPrompt(question, facts) → messages
+ *     4. llm.chat(messages)          → text
+ *     5. parseCitations(text)        — extract bracketed ids
+ *     6. validateCitations(...)      — known vs hallucinated
+ *     7. vault.audit(...)            — record query + facts cited
+ *     8. return { answer, citations, facts, hallucinatedCitations, ... }
+ *
+ * Privacy invariant (§11.2): the engine refuses to call a non-local LLM
+ * unless the caller passes acceptNonLocal: true. This is a hard runtime
+ * gate — every layer downstream of the engine assumes locality.
+ */
+
+"use strict";
+
+const { parseQuery } = require("./query-parser");
+const {
+  buildPrompt,
+  parseCitations,
+  validateCitations,
+  DEFAULT_SYSTEM_PROMPT,
+} = require("./prompt-builder");
+const { toError } = require("./adapter-spec");
+
+const DEFAULT_MAX_FACTS = 80;
+const DEFAULT_MAX_QUERY_LIMIT = 200;
+
+class AnalysisEngine {
+  /**
+   * @param {object} opts
+   * @param {import("./vault").LocalVault} opts.vault
+   * @param {{chat: Function, isLocal: boolean, name?: string}} opts.llm
+   * @param {(question: string, parsed: object) => Promise<Array<{text: string, metadata: object}>>} [opts.ragRetriever]
+   * @param {number} [opts.maxFacts=80]
+   * @param {number} [opts.maxQueryLimit=200]
+   * @param {string} [opts.systemPrompt]
+   */
+  constructor(opts) {
+    if (!opts || typeof opts !== "object") throw new Error("AnalysisEngine: opts required");
+    if (!opts.vault) throw new Error("AnalysisEngine: opts.vault required");
+    if (!opts.llm || typeof opts.llm.chat !== "function") {
+      throw new Error("AnalysisEngine: opts.llm with .chat() required");
+    }
+    if (typeof opts.llm.isLocal !== "boolean") {
+      throw new Error("AnalysisEngine: opts.llm.isLocal must be declared (true/false)");
+    }
+
+    this.vault = opts.vault;
+    this.llm = opts.llm;
+    this.ragRetriever = typeof opts.ragRetriever === "function" ? opts.ragRetriever : null;
+    this.maxFacts = Number.isInteger(opts.maxFacts) && opts.maxFacts > 0 ? opts.maxFacts : DEFAULT_MAX_FACTS;
+    this.maxQueryLimit =
+      Number.isInteger(opts.maxQueryLimit) && opts.maxQueryLimit > 0
+        ? opts.maxQueryLimit
+        : DEFAULT_MAX_QUERY_LIMIT;
+    this.systemPrompt = opts.systemPrompt || DEFAULT_SYSTEM_PROMPT;
+  }
+
+  /**
+   * Ask a natural-language question.
+   *
+   * @param {string} question
+   * @param {object} [options]
+   * @param {boolean} [options.acceptNonLocal=false]  required true for cloud LLMs
+   * @param {number} [options.now]
+   * @param {boolean} [options.skipAudit=false]
+   * @returns {Promise<AskResult>}
+   *
+   * @typedef {object} AskResult
+   * @property {string} answer
+   * @property {string[]} citations            event ids cited AND known
+   * @property {string[]} hallucinatedCitations event ids cited but not in facts
+   * @property {Array<object>} facts           facts handed to the LLM
+   * @property {object} parsed                 parseQuery output
+   * @property {object} usage                  { promptTokens, completionTokens, totalTokens }
+   * @property {string} model
+   * @property {number} durationMs
+   * @property {string|null} warning           "no-facts" | "hallucinated-citations" | null
+   */
+  async ask(question, options = {}) {
+    if (typeof question !== "string" || question.length === 0) {
+      throw new Error("AnalysisEngine.ask: question must be a non-empty string");
+    }
+    if (!this.llm.isLocal && !options.acceptNonLocal) {
+      throw new Error(
+        "AnalysisEngine.ask: LLM declared non-local; pass acceptNonLocal: true to opt in. " +
+          "(Personal Data Hub default policy: all inference stays on-device.)"
+      );
+    }
+
+    const startedAt = Date.now();
+    const parsed = parseQuery(question, { now: options.now });
+
+    // Gather facts from the vault.
+    const facts = this._gatherFacts(parsed);
+
+    // Optional RAG augmentation.
+    let ragContext = [];
+    if (this.ragRetriever) {
+      try {
+        const docs = await this.ragRetriever(question, parsed);
+        if (Array.isArray(docs)) {
+          // RAG retriever returns docs with metadata.id — fetch matching entities
+          // from vault for citation tracking.
+          for (const doc of docs) {
+            if (!doc || !doc.id) continue;
+            const e = this.vault.getEvent(doc.id);
+            if (e && !facts.find((f) => f.id === e.id)) {
+              facts.push(e);
+              ragContext.push(doc.id);
+            }
+          }
+        }
+      } catch (err) {
+        // RAG failure shouldn't abort Q&A — log and continue with direct facts.
+        const e = toError(err, "ragRetriever");
+        try {
+          this.vault.audit("analysis.rag_failed", question, { error: e.message });
+        } catch (_e) {}
+      }
+    }
+
+    // Build prompt.
+    const { messages, factIds, factCount, truncated } = buildPrompt({
+      question,
+      facts,
+      systemPrompt: this.systemPrompt,
+      intent: parsed.intent,
+      timeWindow: parsed.timeWindow,
+      maxFacts: this.maxFacts,
+    });
+
+    // Call LLM.
+    let llmResp;
+    try {
+      llmResp = await this.llm.chat(messages, {
+        temperature: 0.2,
+        purpose: "personal-data-hub.analysis.ask",
+      });
+    } catch (err) {
+      const e = toError(err, "llm.chat");
+      try {
+        this.vault.audit("analysis.llm_failed", question, { error: e.message });
+      } catch (_e) {}
+      throw e;
+    }
+
+    const answer = (llmResp && typeof llmResp.text === "string") ? llmResp.text : "";
+
+    // Parse + validate citations.
+    const cited = parseCitations(answer);
+    const { known, unknown } = validateCitations(cited, factIds);
+
+    // Warnings.
+    let warning = null;
+    if (factCount === 0) warning = "no-facts";
+    else if (unknown.length > 0) warning = "hallucinated-citations";
+
+    const durationMs = Date.now() - startedAt;
+    const usage = llmResp.usage || {};
+
+    if (!options.skipAudit) {
+      try {
+        this.vault.audit("analysis.ask", question, {
+          factCount,
+          truncated,
+          citationsKnown: known.length,
+          citationsUnknown: unknown.length,
+          warning,
+          durationMs,
+          model: this.llm.name || (llmResp && llmResp.model),
+        });
+      } catch (_e) {}
+    }
+
+    return {
+      answer,
+      citations: known,
+      hallucinatedCitations: unknown,
+      facts,
+      ragContextIds: ragContext,
+      parsed,
+      usage,
+      model: this.llm.name || (llmResp && llmResp.model) || "unknown",
+      durationMs,
+      warning,
+    };
+  }
+
+  // ─── Internals ─────────────────────────────────────────────────────
+
+  _gatherFacts(parsed) {
+    // Deliberately do NOT pass parsed.filters.subtype as a vault filter:
+    // the keyword heuristic (`order` vs `payment` vs `transfer`) is too
+    // crude to reliably narrow without false negatives. E.g. a user
+    // asking "在淘宝花了多少" wants taobao-adapter ORDER events; the
+    // keyword parser picks `payment` and would over-filter to zero rows.
+    // Instead we filter by adapter + time window (both reliable) and
+    // pass the subtype/intent into the prompt as a HINT for the LLM to
+    // apply on prose. The LLM is good at filtering; SQL keyword guessing
+    // is brittle.
+    const q = {
+      limit: this.maxQueryLimit,
+    };
+    if (parsed.filters && parsed.filters.adapter) q.adapter = parsed.filters.adapter;
+    if (parsed.timeWindow) {
+      if (Number.isFinite(parsed.timeWindow.since)) q.since = parsed.timeWindow.since;
+      if (Number.isFinite(parsed.timeWindow.until)) q.until = parsed.timeWindow.until;
+    }
+    return this.vault.queryEvents(q);
+  }
+}
+
+module.exports = {
+  AnalysisEngine,
+  DEFAULT_MAX_FACTS,
+  DEFAULT_MAX_QUERY_LIMIT,
+};