@chainlesschain/personal-data-hub 0.1.0 → 0.2.0

package/lib/adapters/shopping-meituan/index.js

@@ -0,0 +1,154 @@
+/**
+ * Phase 7.3b — Meituan (美团) adapter.
+ *
+ * Meituan covers 外卖 (food delivery) + 团购 (group buy) + 酒店 (hotel
+ * — though Ctrip is more common). v0 focuses on 外卖 + 团购 since that's
+ * where most users have the longest history.
+ *
+ * Auth: cookie-based, similar to Taobao/JD.
+ */
+
+"use strict";
+
+const { normalizeOrderRecord, CookieAuth } = require("../shopping-base");
+
+const NAME = "shopping-meituan";
+const VERSION = "0.5.0";
+
+const MEITUAN_ORDERS_URL = "https://h5.meituan.com/order/list";
+
+class MeituanAdapter {
+  constructor(opts = {}) {
+    if (!opts.account || !opts.account.userId) {
+      throw new Error("MeituanAdapter: opts.account.userId required");
+    }
+    this.account = opts.account;
+    this._cookieAuth = new CookieAuth({
+      platform: "meituan",
+      cookies: opts.account.cookies || "",
+    });
+    this._fetchFn = typeof opts.fetchFn === "function" ? opts.fetchFn : defaultFetch;
+
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:cookie-api", "parse:meituan-orders"];
+    this.extractMode = "web-api";
+    this.rateLimits = { perMinute: 8, perDay: 200 };
+    this.dataDisclosure = {
+      fields: ["meituan:orderId / poiName / dishes / totalPrice / deliveryAddress"],
+      sensitivity: "high",
+      legalGate: false,
+    };
+  }
+
+  async authenticate() {
+    const ok = await this._cookieAuth.validate();
+    if (!ok) return { ok: false, reason: "INVALID_COOKIE", error: "cookies missing" };
+    return { ok: true, account: this.account.userId };
+  }
+
+  async healthCheck() {
+    const r = await this.authenticate();
+    return r.ok
+      ? { ok: true, lastChecked: Date.now() }
+      : { ok: false, reason: r.reason, error: r.error };
+  }
+
+  async *sync(opts = {}) {
+    if (!(await this._cookieAuth.validate())) return;
+    const sinceMs = opts.sinceWatermark != null
+      ? parseInt(String(opts.sinceWatermark), 10) || 0
+      : (Date.now() - 365 * 24 * 3600_000);
+    const platforms = opts.platforms || ["waimai", "groupbuy"]; // sub-types
+
+    for (const platform of platforms) {
+      let page = 1;
+      while (true) {
+        const resp = await this._fetchFn({
+          url: MEITUAN_ORDERS_URL,
+          cookies: this._cookieAuth.toHeader(),
+          query: { page, platform },
+        });
+        if (!resp || !Array.isArray(resp.orders)) break;
+        let pageHasNew = false;
+        for (const raw of resp.orders) {
+          const rec = orderToRecord(raw, platform);
+          if (!rec) continue;
+          if (rec.placedAt && rec.placedAt < sinceMs) break;
+          pageHasNew = true;
+          yield {
+            adapter: NAME,
+            originalId: rec.orderId,
+            capturedAt: rec.placedAt || Date.now(),
+            payload: { record: rec, platform },
+          };
+        }
+        if (!pageHasNew || resp.orders.length < 10) break;
+        page += 1;
+      }
+    }
+  }
+
+  normalize(raw) {
+    return normalizeOrderRecord(raw.payload.record, {
+      adapterName: NAME, adapterVersion: VERSION,
+    });
+  }
+}
+
+function orderToRecord(o, platform = "waimai") {
+  if (!o || typeof o !== "object") return null;
+  const orderId = o.orderId || o.viewOrderId || o.id;
+  if (!orderId) return null;
+  const merchant = o.poiName || o.dealName || o.shopName || o.merchantName || "美团";
+
+  const items = [];
+  const dishes = o.dishes || o.dealList || o.itemList || o.items || [];
+  for (const it of dishes) {
+    if (!it) continue;
+    items.push({
+      name: it.name || it.dishName || it.dealName,
+      quantity: parseInt(it.quantity || it.count || 1, 10),
+      unitPrice: parseFloat(it.price || it.unitPrice || 0),
+      sku: it.sku || null,
+    });
+  }
+
+  return {
+    vendorId: "meituan",
+    orderId: String(orderId),
+    placedAt: parseTime(o.orderTime || o.createTime),
+    paidAt: parseTime(o.payTime),
+    status: mapStatus(o.statusDesc || o.statusText || o.status),
+    merchantName: merchant,
+    totalAmount: { value: parseFloat(o.totalPrice || o.totalFee || o.payAmount || 0), currency: "CNY" },
+    items,
+    recipient: o.recipientName,
+    shippingAddress: o.recipientAddress || o.deliveryAddress,
+    extras: { platform, rawStatus: o.statusDesc || o.statusText },
+  };
+}
+
+function parseTime(v) {
+  if (Number.isFinite(v)) return v < 1e12 ? v * 1000 : v;
+  if (typeof v === "string") {
+    const t = Date.parse(v);
+    if (Number.isFinite(t)) return t;
+  }
+  return null;
+}
+
+function mapStatus(s) {
+  const t = String(s || "").toLowerCase();
+  if (t.includes("退款") || t.includes("refund")) return "refunded";
+  if (t.includes("取消") || t.includes("cancel")) return "cancelled";
+  if (t.includes("配送中") || t.includes("派送") || t.includes("shipped")) return "shipped";
+  if (t.includes("已完成") || t.includes("已送达") || t.includes("delivered")) return "delivered";
+  return "placed";
+}
+
+async function defaultFetch(_opts) {
+  throw new Error("MeituanAdapter: no fetchFn configured");
+}
+
+module.exports = { MeituanAdapter, orderToRecord, NAME, VERSION };

package/lib/adapters/shopping-taobao/index.js

@@ -0,0 +1,176 @@
+/**
+ * Phase 7.2 — Taobao + Tmall adapter.
+ *
+ * Auth: cookie-based. User opens taobao.com in a browser, logs in,
+ * copies cookies via the web extension, pastes into hub UI.
+ *
+ * Fetch: Taobao's `/orderList/list.htm` returns a JSON page with order
+ * cards. We use a DI seam `fetchFn` so tests can run on fixture
+ * responses without hitting the network.
+ *
+ * Order schema mapping (Taobao → OrderRecord):
+ *   bizOrderId → orderId
+ *   sellerNick → merchantName
+ *   actualFee → totalAmount.value (yuan, sometimes string)
+ *   mainOrders/subOrders → items[]
+ *   payTime → paidAt (ms epoch)
+ */
+
+"use strict";
+
+const { normalizeOrderRecord, CookieAuth } = require("../shopping-base");
+
+const NAME = "shopping-taobao";
+const VERSION = "0.5.0";
+
+const TAOBAO_ORDERS_URL = "https://h5.m.taobao.com/mlapp/olist.html";
+
+class TaobaoAdapter {
+  constructor(opts = {}) {
+    if (!opts.account || !opts.account.userId) {
+      throw new Error("TaobaoAdapter: opts.account.userId required");
+    }
+    this.account = opts.account;
+    this._cookieAuth = new CookieAuth({
+      platform: "taobao",
+      cookies: opts.account.cookies || "",
+    });
+    this._fetchFn = typeof opts.fetchFn === "function" ? opts.fetchFn : defaultFetch;
+
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:cookie-api", "parse:taobao-orders"];
+    this.extractMode = "web-api";
+    this.rateLimits = { perMinute: 6, perDay: 200 }; // respect Taobao风控
+    this.dataDisclosure = {
+      fields: [
+        "taobao:bizOrderId / sellerNick / items / payTime / actualFee / address",
+      ],
+      sensitivity: "high",
+      legalGate: false,
+    };
+  }
+
+  async authenticate() {
+    const ok = await this._cookieAuth.validate();
+    if (!ok) return { ok: false, reason: "INVALID_COOKIE", error: "cookies missing or empty" };
+    return { ok: true, account: this.account.userId };
+  }
+
+  async healthCheck() {
+    const r = await this.authenticate();
+    return r.ok
+      ? { ok: true, lastChecked: Date.now() }
+      : { ok: false, reason: r.reason, error: r.error };
+  }
+
+  async *sync(opts = {}) {
+    if (!(await this._cookieAuth.validate())) return;
+    const sinceMs = opts.sinceWatermark != null
+      ? parseWatermarkMs(opts.sinceWatermark)
+      : (Date.now() - 365 * 24 * 3600_000); // default last year
+    const pageSize = Number.isFinite(opts.pageSize) ? opts.pageSize : 20;
+    let page = 1;
+    while (true) {
+      const resp = await this._fetchFn({
+        url: TAOBAO_ORDERS_URL,
+        cookies: this._cookieAuth.toHeader(),
+        query: { page, pageSize, ts: Date.now() },
+      });
+      if (!resp || !Array.isArray(resp.orders)) break;
+      let pageHasNew = false;
+      for (const raw of resp.orders) {
+        const rec = orderToRecord(raw);
+        if (!rec) continue;
+        if (rec.placedAt && rec.placedAt < sinceMs) break; // older than watermark
+        pageHasNew = true;
+        yield {
+          adapter: NAME,
+          originalId: rec.orderId,
+          capturedAt: rec.paidAt || rec.placedAt || Date.now(),
+          payload: { record: rec },
+        };
+      }
+      if (!pageHasNew || resp.orders.length < pageSize) break;
+      page += 1;
+    }
+  }
+
+  normalize(raw) {
+    if (!raw || !raw.payload || !raw.payload.record) {
+      throw new Error("TaobaoAdapter.normalize: raw.payload.record missing");
+    }
+    return normalizeOrderRecord(raw.payload.record, {
+      adapterName: NAME,
+      adapterVersion: VERSION,
+    });
+  }
+}
+
+function orderToRecord(o) {
+  if (!o || typeof o !== "object") return null;
+  const orderId = o.bizOrderId || o.orderId || o.id;
+  const merchant = o.sellerNick || o.shopName || o.merchantName;
+  if (!orderId || !merchant) return null;
+  const items = [];
+  const subOrders = o.subOrders || o.itemList || o.items || [];
+  for (const it of subOrders) {
+    if (!it) continue;
+    items.push({
+      name: it.itemTitle || it.title || it.name,
+      quantity: parseInt(it.quantity || it.buyCount || it.num || 1, 10),
+      unitPrice: parseFloat(it.itemPrice || it.unitPrice || it.price || 0),
+      sku: it.skuText || it.sku || null,
+    });
+  }
+  return {
+    vendorId: "taobao",
+    orderId: String(orderId),
+    placedAt: parseTaobaoTime(o.createTime || o.orderCreateTime),
+    paidAt: parseTaobaoTime(o.payTime || o.paidAt),
+    status: mapStatus(o.statusText || o.statusDesc || o.status),
+    merchantName: merchant,
+    totalAmount: { value: parseFloat(o.actualFee || o.payAmount || o.totalAmount || 0), currency: "CNY" },
+    items,
+    recipient: o.receiverName || o.address?.receiverName || null,
+    shippingAddress: o.fullAddress || o.address?.fullAddress || null,
+    trackingNumber: o.trackingNumber || o.expressNo || null,
+    extras: { rawStatus: o.statusText || o.statusDesc, isTmall: !!o.tmallFlag },
+  };
+}
+
+function parseTaobaoTime(v) {
+  if (Number.isFinite(v)) {
+    // Sometimes ms, sometimes seconds (10-digit)
+    return v < 1e12 ? v * 1000 : v;
+  }
+  if (typeof v === "string") {
+    const t = Date.parse(v);
+    if (Number.isFinite(t)) return t;
+  }
+  return null;
+}
+
+function parseWatermarkMs(wm) {
+  if (Number.isFinite(wm)) return wm;
+  const n = parseInt(String(wm), 10);
+  return Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+function mapStatus(s) {
+  const t = String(s || "").toLowerCase();
+  if (t.includes("退款") || t.includes("refund")) return "refunded";
+  if (t.includes("已取消") || t.includes("cancel") || t.includes("已关闭")) return "cancelled";
+  if (t.includes("已发货") || t.includes("shipped")) return "shipped";
+  if (t.includes("已签收") || t.includes("已完成") || t.includes("delivered")) return "delivered";
+  return "placed";
+}
+
+async function defaultFetch(_opts) {
+  // Default: no-op so adapter doesn't accidentally hit real Taobao when
+  // user hasn't configured a fetcher. Production wires a real HTTPS
+  // fetch via the desktop main process (not from renderer).
+  throw new Error("TaobaoAdapter: no fetchFn configured (use a desktop-main wrapper)");
+}
+
+module.exports = { TaobaoAdapter, orderToRecord, parseTaobaoTime, NAME, VERSION };

package/lib/adapters/social-bilibili/index.js

@@ -0,0 +1,171 @@
+/**
+ * Phase 13.1 — Bilibili (B站) adapter.
+ *
+ * Source: B站 Android app stores user data in SQLite (per sjqz/parsers/
+ * social.py BilibiliParser). Phase 7.5 AndroidExtractor pulls the DB
+ * to a local cache; this adapter parses it.
+ *
+ * Tables (sjqz reference):
+ *   - history          watched videos
+ *   - bili_favourite   favorited videos / playlists
+ *   - bili_user        user profile
+ *   - bili_message     私信
+ *
+ * Each row → Event with subtype "browse" (history) / "like" (favorites)
+ * / "message" (DMs) per UnifiedSchema enum.
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const { newId } = require("../../ids");
+
+const NAME = "social-bilibili";
+const VERSION = "0.5.0";
+
+class BilibiliAdapter {
+  constructor(opts = {}) {
+    if (!opts.account || !opts.account.uid) {
+      throw new Error("BilibiliAdapter: opts.account.uid required");
+    }
+    this.account = opts.account;
+    this._dbPath = opts.dbPath || null;
+    this._dbDriverFactory = opts.dbDriverFactory || null;
+
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:sqlite", "parse:bilibili-history", "parse:bilibili-favourite"];
+    this.extractMode = "device-pull";
+    this.rateLimits = {};
+    this.dataDisclosure = {
+      fields: [
+        "bilibili:history (avid / bvid / title / view_at / duration)",
+        "bilibili:favourite (folder / video / save_time)",
+        "bilibili:message (peer / content / time)",
+      ],
+      sensitivity: "medium",
+      legalGate: false,
+    };
+  }
+
+  async authenticate() {
+    return { ok: true, account: this.account.uid };
+  }
+
+  async healthCheck() {
+    return { ok: true, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const dbPath = opts.dbPath || this._dbPath;
+    if (!dbPath || !fs.existsSync(dbPath)) return;
+    const Driver = this._dbDriverFactory
+      ? this._dbDriverFactory()
+      : require("better-sqlite3-multiple-ciphers");
+    const db = new Driver(dbPath, { readonly: true });
+
+    try {
+      const history = trySelect(db, "SELECT * FROM history ORDER BY view_at DESC LIMIT 5000") || [];
+      for (const row of history) {
+        yield {
+          adapter: NAME,
+          originalId: `history-${row.id || row._id || row.kid || row.bvid || row.avid}`,
+          capturedAt: parseTime(row.view_at || row.create_at || row.time),
+          payload: { row, kind: "history" },
+        };
+      }
+
+      const favs = trySelect(db, "SELECT * FROM bili_favourite ORDER BY save_time DESC LIMIT 5000") || [];
+      for (const row of favs) {
+        yield {
+          adapter: NAME,
+          originalId: `fav-${row.id || row.fav_id || row.bvid}`,
+          capturedAt: parseTime(row.save_time || row.time),
+          payload: { row, kind: "favourite" },
+        };
+      }
+    } finally {
+      try { db.close(); } catch (_e) {}
+    }
+  }
+
+  normalize(raw) {
+    if (!raw || !raw.payload || !raw.payload.row) {
+      throw new Error("BilibiliAdapter.normalize: row missing");
+    }
+    const { kind, row } = raw.payload;
+    const now = Date.now();
+    const occurredAt = parseTime(row.view_at || row.save_time || row.create_at || row.time) || now;
+    const source = {
+      adapter: NAME, adapterVersion: VERSION,
+      originalId: raw.originalId, capturedAt: occurredAt,
+      capturedBy: "sqlite",
+    };
+
+    if (kind === "favourite") {
+      return {
+        events: [{
+          id: newId(),
+          type: "event",
+          subtype: "like",
+          occurredAt,
+          actor: "person-self",
+          content: {
+            title: row.title || row.video_title || "(no title)",
+          },
+          ingestedAt: now,
+          source,
+          extra: {
+            bvid: row.bvid || null,
+            avid: row.avid || null,
+            folder: row.folder_name || null,
+            uploader: row.uploader || row.up_name || null,
+          },
+        }],
+        persons: [], places: [], items: [], topics: [],
+      };
+    }
+    // history → browse event
+    return {
+      events: [{
+        id: newId(),
+        type: "event",
+        subtype: "browse",
+        occurredAt,
+        actor: "person-self",
+        content: {
+          title: row.title || row.video_title || "(no title)",
+        },
+        ingestedAt: now,
+        source,
+        extra: {
+          bvid: row.bvid || null,
+          avid: row.avid || null,
+          duration: row.duration || row.progress || null,
+          uploader: row.uploader || row.up_name || null,
+          part: row.part_name || null,
+        },
+      }],
+      persons: [], places: [], items: [], topics: [],
+    };
+  }
+}
+
+function trySelect(db, sql) {
+  try { return db.prepare(sql).all(); } catch (_e) { return null; }
+}
+
+function parseTime(v) {
+  if (Number.isFinite(v)) return v > 1e12 ? v : v * 1000;
+  if (typeof v === "string") {
+    if (/^\d+$/.test(v)) {
+      const n = parseInt(v, 10);
+      return n > 1e12 ? n : n * 1000;
+    }
+    const t = Date.parse(v);
+    return Number.isFinite(t) ? t : null;
+  }
+  return null;
+}
+
+module.exports = { BilibiliAdapter, NAME, VERSION };

package/lib/adapters/social-douyin/index.js

@@ -0,0 +1,116 @@
+/**
+ * Phase 13.3 — Douyin (抖音) adapter — short video platform.
+ *
+ * Source: Douyin Android app SQLite (per sjqz/parsers/douyin.py
+ * DouyinParser). Tables of interest:
+ *   - history / video_history    watched videos
+ *   - favourite / user_favorite  liked / saved
+ *   - search_history             queries
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const { newId } = require("../../ids");
+
+const NAME = "social-douyin";
+const VERSION = "0.5.0";
+
+class DouyinAdapter {
+  constructor(opts = {}) {
+    if (!opts.account || !opts.account.uid) {
+      throw new Error("DouyinAdapter: opts.account.uid required");
+    }
+    this.account = opts.account;
+    this._dbPath = opts.dbPath || null;
+    this._dbDriverFactory = opts.dbDriverFactory || null;
+
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:sqlite", "parse:douyin-history"];
+    this.extractMode = "device-pull";
+    this.rateLimits = {};
+    this.dataDisclosure = {
+      fields: [
+        "douyin:history (aweme_id / title / author / view_time / duration)",
+        "douyin:favourite",
+        "douyin:search_history",
+      ],
+      sensitivity: "medium",
+      legalGate: false,
+    };
+  }
+
+  async authenticate() {
+    return { ok: true, account: this.account.uid };
+  }
+
+  async healthCheck() {
+    return { ok: true, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const dbPath = opts.dbPath || this._dbPath;
+    if (!dbPath || !fs.existsSync(dbPath)) return;
+    const Driver = this._dbDriverFactory
+      ? this._dbDriverFactory()
+      : require("better-sqlite3-multiple-ciphers");
+    const db = new Driver(dbPath, { readonly: true });
+    try {
+      const histories = trySelect(db, "SELECT * FROM video_history ORDER BY view_time DESC LIMIT 5000")
+        || trySelect(db, "SELECT * FROM history ORDER BY view_time DESC LIMIT 5000") || [];
+      for (const row of histories) {
+        yield { adapter: NAME, originalId: `history-${row.id || row.aweme_id}`, capturedAt: parseTime(row.view_time), payload: { row, kind: "history" } };
+      }
+      const favs = trySelect(db, "SELECT * FROM user_favorite ORDER BY create_time DESC LIMIT 5000")
+        || trySelect(db, "SELECT * FROM favourite ORDER BY time DESC LIMIT 5000") || [];
+      for (const row of favs) {
+        yield { adapter: NAME, originalId: `fav-${row.id || row.aweme_id}`, capturedAt: parseTime(row.create_time || row.time), payload: { row, kind: "favourite" } };
+      }
+      const searches = trySelect(db, "SELECT * FROM search_history ORDER BY time DESC LIMIT 5000") || [];
+      for (const row of searches) {
+        yield { adapter: NAME, originalId: `search-${row.id || row._id}`, capturedAt: parseTime(row.time), payload: { row, kind: "search" } };
+      }
+    } finally {
+      try { db.close(); } catch (_e) {}
+    }
+  }
+
+  normalize(raw) {
+    const { kind, row } = raw.payload;
+    const now = Date.now();
+    const occurredAt = parseTime(row.view_time || row.create_time || row.time) || now;
+    const source = { adapter: NAME, adapterVersion: VERSION, originalId: raw.originalId, capturedAt: occurredAt, capturedBy: "sqlite" };
+    const subtypeMap = { history: "browse", favourite: "like", search: "interaction" };
+    return {
+      events: [{
+        id: newId(), type: "event",
+        subtype: subtypeMap[kind] || "browse",
+        occurredAt, actor: "person-self",
+        content: {
+          title: row.title || row.desc || row.keyword || row.query || "(no title)",
+          ...(row.desc && kind !== "search" ? { text: row.desc } : {}),
+        },
+        ingestedAt: now, source,
+        extra: {
+          awemeId: row.aweme_id || null,
+          author: row.author || row.nickname || null,
+          duration: row.duration || null,
+          ...(kind === "search" ? { query: row.keyword || row.query } : {}),
+        },
+      }],
+      persons: [], places: [], items: [], topics: [],
+    };
+  }
+}
+
+function trySelect(db, sql) { try { return db.prepare(sql).all(); } catch (_e) { return null; } }
+function parseTime(v) {
+  if (Number.isFinite(v)) return v > 1e12 ? v : v * 1000;
+  if (typeof v === "string") {
+    if (/^\d+$/.test(v)) { const n = parseInt(v, 10); return n > 1e12 ? n : n * 1000; }
+    const t = Date.parse(v); return Number.isFinite(t) ? t : null;
+  }
+  return null;
+}
+module.exports = { DouyinAdapter, NAME, VERSION };