@chainlesschain/personal-data-hub 0.1.0 → 0.2.0

package/lib/adapters/alipay-bill/counterparty.js

@@ -0,0 +1,129 @@
+/**
+ * Phase 6 — counterparty (交易对方) classifier.
+ *
+ * Design doc §5.5 simplified resolver — full Phase 8 EntityResolver
+ * will replace this with the embedding+LLM pipeline. v0 strategy:
+ *
+ *   1. KNOWN_MERCHANTS membership / substring → "merchant"
+ *   2. Heuristic suffix (公司 / 店 / 服务 / etc.) → "merchant"
+ *   3. 2-4 字纯中文 → "contact" (likely a personal name)
+ *   4. Default → "unknown"
+ *
+ * The "unknown" bucket lets Phase 8 EntityResolver pick these up later.
+ * `needs_resolve: true` is stamped onto Person.extra so a future job can
+ * find them via WHERE clause.
+ */
+
+"use strict";
+
+/**
+ * v1 well-known Chinese consumer merchant whitelist. Covers ~80% of
+ * Alipay transaction counterparties for the typical urban user.
+ * Maintained sorted-ish by category for human readability.
+ */
+const KNOWN_MERCHANTS = new Set([
+  // ── E-commerce ───────────────────────────────────────────────────
+  "淘宝", "天猫", "京东", "京东商城", "拼多多", "苏宁易购", "唯品会",
+  "蘑菇街", "考拉海购", "网易严选", "得物", "小红书", "1号店",
+  "Amazon", "亚马逊",
+  // ── Food / delivery / dining ─────────────────────────────────────
+  "美团", "美团外卖", "饿了么", "大众点评", "盒马", "肯德基", "麦当劳",
+  "星巴克", "瑞幸咖啡", "蜜雪冰城", "海底捞", "Shake Shack",
+  "Costa", "Tim Hortons", "汉堡王", "永和大王", "外婆家", "西贝",
+  // ── Transport / travel ───────────────────────────────────────────
+  "滴滴", "滴滴出行", "曹操出行", "T3 出行", "高德", "高德地图",
+  "百度地图", "12306", "携程", "去哪儿", "同程", "飞猪", "途牛",
+  "驴妈妈", "哈啰", "青桔", "美团单车", "摩拜",
+  // ── Telco / utility ──────────────────────────────────────────────
+  "国家电网", "中国移动", "中国联通", "中国电信", "中国铁通",
+  "燃气公司", "水务局", "自来水公司", "燃气集团",
+  "公积金", "社保",
+  // ── Media / streaming ────────────────────────────────────────────
+  "爱奇艺", "腾讯视频", "优酷", "B站", "哔哩哔哩", "芒果 TV",
+  "网易云音乐", "QQ 音乐", "酷狗", "酷我音乐",
+  // ── Finance / platforms ──────────────────────────────────────────
+  "支付宝", "蚂蚁财富", "余额宝", "花呗", "借呗", "网商银行",
+  "微信支付",
+  // ── Health / pharmacy ────────────────────────────────────────────
+  "京东健康", "阿里健康", "丁香医生", "平安好医生", "美年大健康",
+  // ── Retail brick-and-mortar ──────────────────────────────────────
+  "沃尔玛", "永辉超市", "华润万家", "家乐福", "大润发", "山姆会员店",
+  "便利蜂", "全家", "罗森", "7-Eleven",
+  // ── Apple / Google / SaaS ────────────────────────────────────────
+  "App Store", "Apple", "iCloud", "Google Play",
+  // ── Cosmetics / fashion ──────────────────────────────────────────
+  "屈臣氏", "丝芙兰", "优衣库", "ZARA", "H&M", "Nike", "Adidas",
+  // ── Education / digital ──────────────────────────────────────────
+  "得到", "极客时间", "知乎", "在行", "腾讯课堂", "网易公开课",
+  // ── Government ───────────────────────────────────────────────────
+  "国家税务总局", "税务局", "国家电网", "公安局", "车管所", "民政局",
+]);
+
+// Regex for heuristic suffix matching (company / shop / service words)
+const MERCHANT_SUFFIX_RE = /(公司|集团|有限|股份|店|超市|药房|药店|医院|诊所|学校|学院|大学|加油站|银行|证券|保险|基金|管理处|物业|餐厅|酒店|宾馆|快递|物流|科技)/;
+
+// Person name heuristic: 2-4 Chinese chars, no other text mixed in
+const PERSONAL_NAME_RE = /^[一-龥]{2,4}$/;
+
+// Some Alipay counterparties have prefixes like "**先生(189****1234)" or
+// "***公司 北京分公司" — strip the contact-info tail before classifying.
+function normalizeCounterpartyName(name) {
+  if (typeof name !== "string") return "";
+  return name
+    .replace(/\([^)]*\)/g, "") // () with content
+    .replace(/（[^）]*）/g, "") // Chinese parens
+    .replace(/\*+/g, "") // masked digits
+    .trim();
+}
+
+/**
+ * Classify a counterparty string as merchant / contact / unknown.
+ *
+ * @param {string} rawName
+ * @returns {"merchant"|"contact"|"unknown"}
+ */
+function classifyCounterparty(rawName) {
+  const name = normalizeCounterpartyName(rawName);
+  if (name.length === 0) return "unknown";
+
+  // 1. Exact / substring against known merchants
+  for (const m of KNOWN_MERCHANTS) {
+    if (name.includes(m)) return "merchant";
+  }
+
+  // 2. Suffix heuristic
+  if (MERCHANT_SUFFIX_RE.test(name)) return "merchant";
+
+  // 3. Personal-name heuristic
+  if (PERSONAL_NAME_RE.test(name)) return "contact";
+
+  return "unknown";
+}
+
+/**
+ * Get a stable Person.id for a counterparty so repeat imports dedup
+ * by name. Phase 8 EntityResolver may later merge multiple ids into
+ * one — but for v0 same-name → same-id is the right default.
+ */
+function counterpartyToPersonId(rawName) {
+  const name = normalizeCounterpartyName(rawName);
+  // Keep ids URL-safe and stable. Hash via a simple normalize so accents
+  // and whitespace variations collapse. v0 just uses the trimmed name
+  // since Alipay counterparty strings are already canonical.
+  return `person-alipay-${slugify(name)}`;
+}
+
+function slugify(s) {
+  return String(s || "")
+    .toLowerCase()
+    .replace(/\s+/g, "-")
+    .replace(/[^\w一-鿿-]/g, "")
+    .slice(0, 80);
+}
+
+module.exports = {
+  KNOWN_MERCHANTS,
+  classifyCounterparty,
+  counterpartyToPersonId,
+  normalizeCounterpartyName,
+};

package/lib/adapters/alipay-bill/csv-parser.js

@@ -0,0 +1,217 @@
+/**
+ * Phase 6 — Alipay 账单 CSV 解析器
+ *
+ * 支付宝 "开具交易流水证明" 导出的 CSV 格式（GBK 默认，新版部分 UTF-8 BOM）：
+ *
+ *   行 1   `支付宝交易记录明细查询`
+ *   行 2   `账号:[email@example.com / 13800001111]`
+ *   行 3   `起始日期:[2024-04-01 00:00:00]    终止日期:[2024-05-01 00:00:00]`
+ *   行 4   `-------------------交易记录明细列表-------------------`
+ *   行 5   `交易号,商家订单号,交易创建时间,付款时间,...` ← header
+ *   行 6+  数据行
+ *   末尾   `-------------------交易记录明细列表结束-------------------`
+ *   再    汇总文本（"导出时间"、"用户姓名" 等元数据）— 跳过
+ *
+ * 设计选择：
+ *   1. 手写 parser（不引 csv-parse）。Alipay CSV 字段都用半角逗号，
+ *      字段内不嵌逗号（商品名含逗号也会被 Alipay 转义为中文 ， 或省略），
+ *      Naive split 已足够，单测覆盖 50+ 真实样本。
+ *   2. 编码：先尝 UTF-8 decode 看是否含合理的中文 magic 字符串
+ *      ("交易号" / "支付宝")；含 → UTF-8；否则降级 GBK（via iconv-lite）。
+ *   3. 终止：碰到 "交易记录明细列表结束" 或下一个非数据行（不含逗号或
+ *      首字段不是 yyyy 开头）。
+ *
+ * 返回 `{ header: {...meta}, rows: [...] }`：
+ *   - header.account     `email@example.com` 或手机
+ *   - header.startDate   ISO-ish string
+ *   - header.endDate
+ *   - rows               RawTransaction 数组（design doc §5.3 形状）
+ */
+
+"use strict";
+
+/** @typedef {import('./types').RawTransaction} RawTransaction */
+
+const FIELD_ORDER = [
+  "txId",
+  "merchantOrderNumber",
+  "createdAt",
+  "paidAt",
+  "lastModifiedAt",
+  "sourceChannel",
+  "alipayType",
+  "counterparty",
+  "itemName",
+  "amount",
+  "direction",
+  "status",
+  "serviceFee",
+  "refundedAmount",
+  "note",
+  "fundStatus",
+];
+
+const MAGIC_HEADER_ROW = "交易号"; // header line starts with this
+
+/**
+ * Decode a Buffer using UTF-8 first, falling back to GBK via iconv-lite.
+ *
+ * @param {Buffer} buf
+ * @param {{ iconvImpl?: Function }} [opts]   inject for tests
+ * @returns {{ text: string, encoding: string }}
+ */
+function decodeBuffer(buf, opts = {}) {
+  if (!Buffer.isBuffer(buf)) {
+    throw new Error("decodeBuffer: Buffer required");
+  }
+  // Strip BOM if present (UTF-8 BOM = EF BB BF)
+  let work = buf;
+  if (buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) {
+    work = buf.slice(3);
+  }
+  const utf8 = work.toString("utf-8");
+  // UTF-8 confidence check: does it contain expected Alipay header tokens?
+  if (utf8.includes("交易号") || utf8.includes("支付宝交易记录")) {
+    return { text: utf8, encoding: "utf-8" };
+  }
+  // Fall back to GBK
+  const iconv = typeof opts.iconvImpl === "function" ? opts.iconvImpl : loadIconvLite();
+  const decoded = iconv(work, "gbk");
+  return { text: decoded, encoding: "gbk" };
+}
+
+let _iconvCache = null;
+function loadIconvLite() {
+  if (_iconvCache) return _iconvCache;
+  try {
+    // eslint-disable-next-line global-require
+    const il = require("iconv-lite");
+    _iconvCache = (buf, enc) => il.decode(buf, enc);
+  } catch (err) {
+    throw new Error(
+      `iconv-lite not installed — Alipay CSV needs it for GBK decode. ${err && err.message ? err.message : err}`,
+    );
+  }
+  return _iconvCache;
+}
+
+/**
+ * Parse a decoded CSV text → { header, rows }.
+ *
+ * @param {string} text
+ * @returns {{ header: object, rows: RawTransaction[] }}
+ */
+function parseAlipayCsv(text) {
+  if (typeof text !== "string" || text.length === 0) {
+    return { header: {}, rows: [] };
+  }
+  const lines = text.split(/\r?\n/);
+  const header = {};
+
+  // ── Step 1: scan preamble for account + date range, then find header row idx
+  let headerIdx = -1;
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i];
+    // Match account: 账号:[email@... / phone]
+    const acctMatch = line.match(/账号\s*:?\s*\[?([^\]\s]+@[^\]\s]+|\d{11})\]?/);
+    if (acctMatch) header.account = acctMatch[1];
+    // Match date range: 起始日期:[2024-04-01 00:00:00] 终止日期:[2024-05-01 00:00:00]
+    const startMatch = line.match(/起始日期\s*:?\s*\[?([\d-]+\s+[\d:]+)\]?/);
+    if (startMatch) header.startDate = startMatch[1];
+    const endMatch = line.match(/终止日期\s*:?\s*\[?([\d-]+\s+[\d:]+)\]?/);
+    if (endMatch) header.endDate = endMatch[1];
+    // Detect the column-header line
+    if (line.startsWith(MAGIC_HEADER_ROW)) {
+      headerIdx = i;
+      break;
+    }
+  }
+  if (headerIdx === -1) {
+    // No "交易号" header line — file is malformed / empty / not an Alipay CSV
+    return { header, rows: [], warning: "header row '交易号,...' not found" };
+  }
+
+  // ── Step 2: parse rows after headerIdx until terminator or non-data line
+  const rows = [];
+  for (let i = headerIdx + 1; i < lines.length; i += 1) {
+    const line = lines[i];
+    if (!line) continue;
+    if (line.includes("交易记录明细列表结束") || line.includes("---")) break;
+    // A data line should have ≥ 12 commas (16 fields). Otherwise it's
+    // probably trailing metadata like "导出时间:..."
+    const commas = (line.match(/,/g) || []).length;
+    if (commas < 10) continue;
+
+    const fields = splitCsvLine(line);
+    if (fields.length < FIELD_ORDER.length) {
+      // Lenient: pad with empty strings to match the schema
+      while (fields.length < FIELD_ORDER.length) fields.push("");
+    }
+    const row = {};
+    for (let j = 0; j < FIELD_ORDER.length; j += 1) {
+      row[FIELD_ORDER[j]] = fields[j] != null ? fields[j].trim() : "";
+    }
+    // Skip empty-id rows
+    if (!row.txId) continue;
+    rows.push(row);
+  }
+
+  return { header, rows };
+}
+
+/**
+ * Lightweight CSV-line split. Alipay rows don't quote fields, so a plain
+ * `,` split is correct in practice. We still tolerate double-quoted
+ * fields just in case (`"abc, def"`) for forward-compat.
+ *
+ * Exported for unit tests.
+ */
+function splitCsvLine(line) {
+  if (!line.includes('"')) {
+    return line.split(",");
+  }
+  // Quoted-field aware split
+  const out = [];
+  let cur = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i];
+    if (ch === '"') {
+      if (inQuotes && line[i + 1] === '"') {
+        cur += '"';
+        i += 1; // escaped quote
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (ch === "," && !inQuotes) {
+      out.push(cur);
+      cur = "";
+    } else {
+      cur += ch;
+    }
+  }
+  out.push(cur);
+  return out;
+}
+
+/**
+ * Top-level: take a raw Buffer (the CSV file bytes, ZIP-decompressed by
+ * zip-decryptor.js) and return parsed rows + metadata.
+ *
+ * @param {Buffer} buf
+ * @param {{ iconvImpl?: Function }} [opts]
+ * @returns {{ encoding: string, header: object, rows: RawTransaction[] }}
+ */
+function parseAlipayCsvBuffer(buf, opts = {}) {
+  const { text, encoding } = decodeBuffer(buf, opts);
+  const parsed = parseAlipayCsv(text);
+  return { encoding, ...parsed };
+}
+
+module.exports = {
+  parseAlipayCsv,
+  parseAlipayCsvBuffer,
+  decodeBuffer,
+  splitCsvLine,
+  FIELD_ORDER,
+};

package/lib/adapters/alipay-bill/index.js

@@ -0,0 +1,41 @@
+"use strict";
+
+const {
+  AlipayBillAdapter,
+  mapAlipayTypeToSubtype,
+  parseAlipayDateTime,
+  NAME,
+  VERSION,
+} = require("./alipay-bill-adapter");
+const {
+  parseAlipayCsv,
+  parseAlipayCsvBuffer,
+  decodeBuffer,
+  splitCsvLine,
+  FIELD_ORDER,
+} = require("./csv-parser");
+const { extractCsvFromZip } = require("./zip-decryptor");
+const {
+  KNOWN_MERCHANTS,
+  classifyCounterparty,
+  counterpartyToPersonId,
+  normalizeCounterpartyName,
+} = require("./counterparty");
+
+module.exports = {
+  AlipayBillAdapter,
+  ALIPAY_BILL_NAME: NAME,
+  ALIPAY_BILL_VERSION: VERSION,
+  mapAlipayTypeToSubtype,
+  parseAlipayDateTime,
+  parseAlipayCsv,
+  parseAlipayCsvBuffer,
+  decodeAlipayBuffer: decodeBuffer,
+  splitAlipayCsvLine: splitCsvLine,
+  ALIPAY_CSV_FIELDS: FIELD_ORDER,
+  extractAlipayCsvFromZip: extractCsvFromZip,
+  ALIPAY_KNOWN_MERCHANTS: KNOWN_MERCHANTS,
+  classifyAlipayCounterparty: classifyCounterparty,
+  alipayCounterpartyToPersonId: counterpartyToPersonId,
+  normalizeAlipayCounterpartyName: normalizeCounterpartyName,
+};

package/lib/adapters/alipay-bill/zip-decryptor.js

@@ -0,0 +1,111 @@
+/**
+ * Phase 6 — Alipay 加密 ZIP 解压器
+ *
+ * 支付宝官方导出的 `alipay_record_*.zip` 用 ZipCrypto（传统 PKZIP 密码），
+ * 默认密码 = 用户身份证后 6 位。adm-zip 0.5+ 内置 ZipCrypto 解密。
+ *
+ * 使用：
+ *   const { extractCsvFromZip } = require('./zip-decryptor');
+ *   const csvBuf = await extractCsvFromZip(zipPath, { password: "123456" });
+ *   // csvBuf 是 Buffer，再交给 parseAlipayCsvBuffer
+ *
+ * 失败模式：
+ *   - 文件不存在 → throws { code: "ENOENT" }
+ *   - 不是 ZIP → throws "not a valid zip"
+ *   - 密码错误 → throws "Wrong Password" (adm-zip 自带错误消息)
+ *   - ZIP 内没有 .csv → throws "no CSV file in ZIP"
+ *
+ * 全部 throws 是因为这个层只做"打开 + 解压"动作，错误分类放到 adapter
+ * authenticate / sync 路径处理（统一映射到 PersonalDataAdapter 协议）。
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+
+/**
+ * Extract the first .csv file from an Alipay ZIP. Returns its raw Buffer.
+ *
+ * @param {string} zipPath
+ * @param {object} [opts]
+ * @param {string} [opts.password]
+ * @param {Function} [opts.admZipImpl]   DI seam: a constructor function with
+ *                                       the adm-zip API (new AdmZip(path)).
+ *                                       Defaults to `require("adm-zip")`.
+ * @returns {Promise<{ buffer: Buffer, filename: string }>}
+ */
+async function extractCsvFromZip(zipPath, opts = {}) {
+  if (typeof zipPath !== "string" || zipPath.length === 0) {
+    throw new Error("extractCsvFromZip: zipPath required");
+  }
+  // Surface ENOENT cleanly
+  if (!fs.existsSync(zipPath)) {
+    const err = new Error(`ZIP file not found: ${zipPath}`);
+    err.code = "ENOENT";
+    throw err;
+  }
+
+  const AdmZip = typeof opts.admZipImpl === "function"
+    ? opts.admZipImpl
+    : loadAdmZip();
+
+  let zip;
+  try {
+    zip = new AdmZip(zipPath);
+  } catch (err) {
+    throw new Error(
+      `Failed to open ZIP: ${err && err.message ? err.message : err}`,
+    );
+  }
+
+  const entries = zip.getEntries();
+  const csvEntry = entries.find((e) => /\.csv$/i.test(e.entryName));
+  if (!csvEntry) {
+    throw new Error(
+      `No CSV file in ZIP; found: ${entries.map((e) => e.entryName).join(", ") || "(empty)"}`,
+    );
+  }
+
+  // adm-zip's password-aware extract: `readFile(entry, password)`.
+  // For unencrypted ZIPs the password is ignored.
+  let csvBuffer;
+  try {
+    csvBuffer = zip.readFile(csvEntry, opts.password || "");
+  } catch (err) {
+    // adm-zip throws strings sometimes; wrap.
+    const msg = err && err.message ? err.message : String(err);
+    if (/password/i.test(msg) || /wrong/i.test(msg)) {
+      const e = new Error(`ZIP password incorrect or missing`);
+      e.code = "ZIP_PASSWORD_FAILED";
+      throw e;
+    }
+    throw new Error(`ZIP extract failed: ${msg}`);
+  }
+
+  if (!Buffer.isBuffer(csvBuffer) || csvBuffer.length === 0) {
+    // adm-zip returns null on password failure in some versions
+    const e = new Error("ZIP password incorrect (empty buffer returned)");
+    e.code = "ZIP_PASSWORD_FAILED";
+    throw e;
+  }
+
+  return { buffer: csvBuffer, filename: csvEntry.entryName };
+}
+
+let _admZipCache = null;
+function loadAdmZip() {
+  if (_admZipCache) return _admZipCache;
+  try {
+    // eslint-disable-next-line global-require
+    _admZipCache = require("adm-zip");
+  } catch (err) {
+    throw new Error(
+      `adm-zip not installed — Phase 6 needs it. ${err && err.message ? err.message : err}`,
+    );
+  }
+  return _admZipCache;
+}
+
+module.exports = {
+  extractCsvFromZip,
+};