@chainlesschain/personal-data-hub 0.3.1 → 0.3.6

package/lib/adapters/social-douyin-adb/snapshot-builder.js

@@ -0,0 +1,174 @@
+"use strict";
+
+/**
+ * Phase 2a (Douyin C 路径 — 2026-05-25): IM-db parse result → snapshot JSON.
+ *
+ * Takes the `{messages, contacts}` shape from im-db-parser.js and produces
+ * a snapshot matching the existing `social-douyin` adapter's
+ * SNAPSHOT_SCHEMA_VERSION=1 contract — so we reuse the adapter's snapshot
+ * mode (`_syncViaSnapshot`) instead of opening a second adapter.
+ *
+ * Mirrors social-bilibili-adb/snapshot-builder.js. Single-source-of-truth
+ * for the adapter; we feed it via different upstreams.
+ *
+ * Snapshot schema (matches social-douyin/index.js:SNAPSHOT_SCHEMA_VERSION):
+ *
+ *   {
+ *     "schemaVersion": 1,
+ *     "snapshottedAt": <epoch-ms>,
+ *     "account": {
+ *       "secUid": null,             // C 路径不调 X-Bogus profile, 不知 secUid
+ *       "shortId": null,
+ *       "displayName": ""
+ *     },
+ *     "events": [
+ *       { "kind": "message", "id": "msg-<conv>-<time>", "capturedAt": <ms>,
+ *         "senderUid": "...", "conversationId": "...",
+ *         "text": "...", "readStatus": 0/1, "contentBlob": "..." },
+ *       { "kind": "contact", "id": "contact-<uid>", "capturedAt": <ms>,
+ *         "uid": "...", "shortId": "...", "name": "...",
+ *         "avatarUrl": "...", "followStatus": 0/1/2 }
+ *     ]
+ *   }
+ *
+ * Note: Douyin IM doesn't have a "this is me" marker — the db includes
+ * messages where `senderUid === <db-filename-uid>` (sent by self) and
+ * `senderUid !== <db-filename-uid>` (received). Both go into the snapshot;
+ * the consumer (e.g. PDH search) can filter by senderUid if needed.
+ */
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+const crypto = require("node:crypto");
+
+const SNAPSHOT_SCHEMA_VERSION = 1;
+
+/**
+ * Build an in-memory snapshot from parsed IM db rows. Pure function — no
+ * disk IO.
+ *
+ * @param {{
+ *   uid: string,
+ *   messages?: Array,
+ *   contacts?: Array,
+ *   snapshottedAt?: number,
+ *   displayName?: string,
+ * }} input
+ * @returns {{schemaVersion: number, snapshottedAt: number, account: object, events: Array}}
+ */
+function buildSnapshot(input) {
+  if (!input || typeof input !== "object") {
+    throw new TypeError("buildSnapshot: input must be an object");
+  }
+  const uid = input.uid;
+  if (typeof uid !== "string" || uid.length === 0) {
+    throw new TypeError("buildSnapshot: input.uid must be a non-empty string");
+  }
+  const snapshottedAt =
+    Number.isFinite(input.snapshottedAt) && input.snapshottedAt > 0
+      ? input.snapshottedAt
+      : Date.now();
+  const account = {
+    // secUid / shortId unknown via pure-db extraction (those live in the
+    // app's webview cookies / passport endpoint). Leave null so consumers
+    // know not to use them as canonical IDs.
+    secUid: null,
+    shortId: uid, // Douyin numeric uid is the shortId equivalent
+    displayName:
+      typeof input.displayName === "string" ? input.displayName : "",
+  };
+  const events = [];
+
+  // messages
+  const messages = Array.isArray(input.messages) ? input.messages : [];
+  messages.forEach((m, idx) => {
+    if (!m || typeof m !== "object") return;
+    const capturedAt =
+      typeof m.createdTimeMs === "number" && m.createdTimeMs > 0
+        ? m.createdTimeMs
+        : snapshottedAt;
+    // ID strategy: conversationId + createdTime is a stable composite
+    // key (both required by Douyin's IM protocol). Fallback to senderUid
+    // + time for very old rows that pre-date conversation_id.
+    const idPart =
+      m.conversationId && m.createdTimeMs
+        ? `${m.conversationId}-${m.createdTimeMs}`
+        : m.senderUid && m.createdTimeMs
+          ? `${m.senderUid}-${m.createdTimeMs}`
+          : `msg-${idx}`;
+    events.push({
+      kind: "message",
+      id: `msg-${idPart}`,
+      capturedAt,
+      senderUid: m.senderUid || null,
+      conversationId: m.conversationId || null,
+      text: m.text || null,
+      readStatus: typeof m.readStatus === "number" ? m.readStatus : null,
+      contentBlob: m.contentBlob || null,
+    });
+  });
+
+  // contacts
+  const contacts = Array.isArray(input.contacts) ? input.contacts : [];
+  contacts.forEach((c, idx) => {
+    if (!c || typeof c !== "object") return;
+    events.push({
+      kind: "contact",
+      id: c.uid ? `contact-${c.uid}` : `contact-${idx}`,
+      capturedAt: snapshottedAt, // SIMPLE_USER has no per-row timestamp
+      uid: c.uid || null,
+      shortId: c.shortId || null,
+      name: c.name || null,
+      avatarUrl: c.avatarUrl || null,
+      followStatus:
+        typeof c.followStatus === "number" ? c.followStatus : null,
+    });
+  });
+
+  return {
+    schemaVersion: SNAPSHOT_SCHEMA_VERSION,
+    snapshottedAt,
+    account,
+    events,
+  };
+}
+
+/**
+ * Write a snapshot to disk as JSON. Default destination is
+ * `<os.tmpdir()>/cc-douyin-snapshot-<uuid>.json`. Returns the absolute
+ * path. Caller is responsible for cleanup.
+ */
+function writeSnapshotJson(snapshot, opts = {}) {
+  const dir = opts.dir || os.tmpdir();
+  const fileName =
+    opts.fileName || `cc-douyin-snapshot-${crypto.randomUUID()}.json`;
+  if (fileName.includes("/") || fileName.includes("\\")) {
+    throw new Error(
+      "writeSnapshotJson: opts.fileName must be a basename, not a path",
+    );
+  }
+  const full = path.join(dir, fileName);
+  fs.writeFileSync(full, JSON.stringify(snapshot), "utf-8");
+  return full;
+}
+
+/**
+ * Best-effort delete of a snapshot file. Used in finally blocks; never
+ * throws.
+ */
+function cleanupSnapshotJson(filePath) {
+  if (!filePath) return;
+  try {
+    fs.unlinkSync(filePath);
+  } catch (_e) {
+    // ignore
+  }
+}
+
+module.exports = {
+  buildSnapshot,
+  writeSnapshotJson,
+  cleanupSnapshotJson,
+  SNAPSHOT_SCHEMA_VERSION,
+};

package/lib/adapters/social-weibo-adb/api-client.js

@@ -0,0 +1,281 @@
+"use strict";
+
+/**
+ * Phase 3a (Weibo C 路径 — 2026-05-25): Node-side WeiboApiClient.
+ *
+ * Byte-parity port of
+ * `android-app/.../pdh/social/weibo/WeiboApiClient.kt` for the desktop
+ * PC + ADB path. Same m.weibo.cn endpoints, same headers, same JSON
+ * parse shape. Lockstep with the Kotlin version — if a real-device trap
+ * surfaces fix both sides.
+ *
+ * **Key differences from Bilibili Phase 1b**:
+ *  1. **No WBI signing** — m.weibo.cn mobile API requires cookie + UA +
+ *     XHR header but no signature. Simpler client, no /nav handshake.
+ *  2. **UID via /api/config** — Weibo cookie has no DedeUserID equivalent;
+ *     fetchUid() must do an HTTP roundtrip and persist the result.
+ *  3. **Time field is ISO 8601** — "Sun Jan 12 13:45:00 +0800 2026"
+ *     format (not unix seconds like Bilibili). Java's SimpleDateFormat
+ *     parses it; Node's Date can too once we know the format.
+ *  4. **Timeline endpoint via containerid** — user posts go through
+ *     /api/container/getIndex?containerid=107603<uid>, not a dedicated
+ *     /api/posts.
+ *  5. **Anti-bot signal**: missing `X-Requested-With: XMLHttpRequest` +
+ *     `MWeibo-Pwa: 1` → 30x redirect to login HTML.
+ *
+ * 4 endpoints:
+ *   - config     /api/config (fetchUid + login state check)
+ *   - posts      /api/container/getIndex?type=uid&value=<uid>&containerid=107603<uid>
+ *   - favourites /api/favorites?page=1
+ *   - follows    /api/friendships/friends?uid=<uid>&page=1
+ *
+ * Errors don't throw — endpoints that fail return [] and lastErrorCode +
+ * lastErrorMessage surface the cause for partial-result diagnostics.
+ */
+
+const DEFAULT_BASE_URL = "https://m.weibo.cn/";
+
+// Pinned Chrome 120 mobile UA — must look like a browser, default
+// `node-fetch/x.y.z` returns -100 silentband.
+const BROWSER_UA =
+  "Mozilla/5.0 (Linux; Android 14; ChainlessChain) AppleWebKit/537.36 " +
+  "(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36";
+
+const BROWSER_HEADERS = Object.freeze({
+  "User-Agent": BROWSER_UA,
+  Referer: "https://m.weibo.cn/",
+  Accept: "application/json, text/plain, */*",
+  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+  // m.weibo.cn anti-bot: missing these → HTML redirect, not JSON
+  "X-Requested-With": "XMLHttpRequest",
+  "MWeibo-Pwa": "1",
+});
+
+/**
+ * Parse Weibo's ISO-8601-ish timestamp.
+ *   "Sun Jan 12 13:45:00 +0800 2026" → epoch ms
+ *   "1716383021"                     → epoch ms (× 1000 since it's < 1e12)
+ *   "1716383021000"                  → epoch ms (verbatim)
+ *
+ * Mirrors WeiboApiClient.kt:parseWeiboTime.
+ */
+function parseWeiboTime(raw) {
+  if (typeof raw !== "string" || raw.length === 0) return 0;
+  // Digits-only fallback — Weibo occasionally serves unix-seconds verbatim
+  if (/^\d+$/.test(raw)) {
+    const n = parseInt(raw, 10);
+    return n > 1e12 ? n : n * 1000;
+  }
+  // "EEE MMM dd HH:mm:ss Z yyyy" — JS Date.parse handles this in V8 / Node.
+  const t = Date.parse(raw);
+  return Number.isFinite(t) ? t : 0;
+}
+
+/**
+ * Strip HTML from Weibo's `text` field (contains <a>, <span>, &nbsp; etc.).
+ * Mirrors WeiboApiClient.kt:stripHtml.
+ */
+function stripHtml(raw) {
+  if (typeof raw !== "string" || raw.length === 0) return "";
+  return raw
+    .replace(/<[^>]+>/g, "")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .trim();
+}
+
+class WeiboApiClient {
+  constructor(opts = {}) {
+    this.baseUrl = opts.baseUrl || DEFAULT_BASE_URL;
+    if (!this.baseUrl.endsWith("/")) this.baseUrl += "/";
+    this._fetch = opts.fetch || globalThis.fetch;
+    if (typeof this._fetch !== "function") {
+      throw new Error(
+        "WeiboApiClient: fetch not available — pass opts.fetch or run on Node 18+",
+      );
+    }
+    this.lastErrorCode = 0;
+    this.lastErrorMessage = null;
+  }
+
+  /**
+   * GET <url> with browser-like headers. Mirrors Kotlin doGetJson —
+   * including the non-JSON-body check (Weibo redirects to login HTML
+   * when cookie expired).
+   */
+  async _doGetJson(url, cookie) {
+    try {
+      const resp = await this._fetch(url.toString(), {
+        method: "GET",
+        headers: { ...BROWSER_HEADERS, Cookie: cookie },
+      });
+      const body = await resp.text();
+      if (!resp.ok) {
+        this._setLastError(resp.status, `HTTP ${resp.status}`);
+        return null;
+      }
+      const trimmed = body.trimStart();
+      if (!trimmed.startsWith("{")) {
+        // Login redirect / anti-bot HTML — cookie expired or anti-spider hit
+        this._setLastError(-4, "non-json (cookie expired?)");
+        return null;
+      }
+      let obj;
+      try {
+        obj = JSON.parse(body);
+      } catch (e) {
+        this._setLastError(-3, "parse: " + (e.message || String(e)));
+        return null;
+      }
+      const ok = typeof obj.ok === "number" ? obj.ok : 1;
+      if (ok !== 1) {
+        this._setLastError(ok, (obj.msg || "").toString());
+        return null;
+      }
+      this._clearLastError();
+      return obj;
+    } catch (e) {
+      this._setLastError(-2, "IO: " + (e.message || String(e)));
+      return null;
+    }
+  }
+
+  _setLastError(code, message) {
+    this.lastErrorCode = code;
+    this.lastErrorMessage = message;
+  }
+  _clearLastError() {
+    this.lastErrorCode = 0;
+    this.lastErrorMessage = null;
+  }
+
+  /**
+   * Fetch /api/config to get UID + validate login state. Returns numeric
+   * UID on success, null on failure (cookie expired / not logged in).
+   * Mirrors WeiboApiClient.kt:fetchUid.
+   */
+  async fetchUid(cookie) {
+    const url = new URL("api/config", this.baseUrl);
+    const obj = await this._doGetJson(url, cookie);
+    if (!obj) return null;
+    const data = obj.data || {};
+    if (!data.login) return null;
+    const uidStr = data.uid;
+    const uid = parseInt(uidStr, 10);
+    return Number.isFinite(uid) && uid > 0 ? uid : null;
+  }
+
+  /**
+   * Fetch the user's own posts (timeline). Mirrors fetchPosts —
+   * containerid=107603<uid> is the magic "user's own mblog" container.
+   */
+  async fetchPosts(cookie, uid, opts = {}) {
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 100;
+    const containerid = `107603${uid}`;
+    const url = new URL("api/container/getIndex", this.baseUrl);
+    url.searchParams.set("type", "uid");
+    url.searchParams.set("value", String(uid));
+    url.searchParams.set("containerid", containerid);
+    const obj = await this._doGetJson(url, cookie);
+    if (!obj) return [];
+    const data = obj.data || {};
+    const cards = Array.isArray(data.cards) ? data.cards : [];
+    const out = [];
+    for (const card of cards) {
+      if (out.length >= limit) break;
+      if (!card || card.card_type !== 9) continue; // card_type=9 = mblog
+      const blog = card.mblog;
+      if (!blog) continue;
+      const mid = (blog.mid && String(blog.mid)) || (blog.id && String(blog.id));
+      if (!mid) continue;
+      out.push({
+        mid,
+        text: stripHtml(blog.text),
+        createdAt: parseWeiboTime(blog.created_at),
+        source: blog.source || null,
+        repostsCount: typeof blog.reposts_count === "number" ? blog.reposts_count : 0,
+        commentsCount:
+          typeof blog.comments_count === "number" ? blog.comments_count : 0,
+        likesCount:
+          typeof blog.attitudes_count === "number" ? blog.attitudes_count : 0,
+        picCount: typeof blog.pic_num === "number" ? blog.pic_num : 0,
+      });
+    }
+    return out;
+  }
+
+  /** Mirrors fetchFavourites. */
+  async fetchFavourites(cookie, opts = {}) {
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 100;
+    const url = new URL("api/favorites", this.baseUrl);
+    url.searchParams.set("page", "1");
+    const obj = await this._doGetJson(url, cookie);
+    if (!obj) return [];
+    const data = obj.data || {};
+    const favs = Array.isArray(data.favorites) ? data.favorites : [];
+    const out = [];
+    for (let i = 0; i < Math.min(limit, favs.length); i++) {
+      const fav = favs[i];
+      if (!fav) continue;
+      const status = fav.status;
+      if (!status) continue;
+      const mid = (status.mid && String(status.mid)) || (status.id && String(status.id));
+      if (!mid) continue;
+      const author = status.user || {};
+      const favAt =
+        parseWeiboTime(fav.favorited_time) ||
+        parseWeiboTime(status.created_at) ||
+        0;
+      out.push({
+        mid,
+        text: stripHtml(status.text),
+        favAt,
+        authorScreenName: author.screen_name || null,
+      });
+    }
+    return out;
+  }
+
+  /** Mirrors fetchFollows. */
+  async fetchFollows(cookie, uid, opts = {}) {
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 200;
+    const url = new URL("api/friendships/friends", this.baseUrl);
+    url.searchParams.set("uid", String(uid));
+    url.searchParams.set("page", "1");
+    const obj = await this._doGetJson(url, cookie);
+    if (!obj) return [];
+    const data = obj.data || {};
+    const users = Array.isArray(data.users) ? data.users : [];
+    const out = [];
+    for (let i = 0; i < Math.min(limit, users.length); i++) {
+      const u = users[i];
+      if (!u) continue;
+      const followUid = typeof u.id === "number" ? u.id : 0;
+      if (followUid === 0) continue;
+      out.push({
+        uid: followUid,
+        screenName: u.screen_name || "(unnamed)",
+        description: u.description || null,
+        avatarUrl: u.profile_image_url || null,
+        // m.weibo.cn /api/friendships/friends doesn't return follow_time —
+        // 0 lets the snapshot builder fall back to snapshottedAt.
+        followedAt: 0,
+      });
+    }
+    return out;
+  }
+}
+
+module.exports = {
+  WeiboApiClient,
+  // Exposed for tests
+  _internals: {
+    parseWeiboTime,
+    stripHtml,
+    BROWSER_UA,
+    BROWSER_HEADERS,
+  },
+};

package/lib/adapters/social-weibo-adb/collector.js

@@ -0,0 +1,169 @@
+"use strict";
+
+/**
+ * Phase 3a (Weibo C 路径 — 2026-05-25): end-to-end orchestrator.
+ *
+ *   bridge.invoke("weibo.cookies")          ← Phase 3a cookies extension
+ *           │
+ *           ▼  {cookie, diagnostic}
+ *   WeiboApiClient.fetchUid                  ← /api/config 拿 UID + 验登录
+ *           │
+ *           ▼  uid (numeric)
+ *   fetchPosts + fetchFavourites + fetchFollows   (partial-failure OK)
+ *           │
+ *           ▼  3 arrays
+ *   buildSnapshot + writeSnapshotJson        ← schemaVersion=1
+ *           │
+ *           ▼
+ *   registry.syncAdapter("social-weibo", { inputPath })
+ *
+ * Mirror of social-bilibili-adb/collector.js — same `{ok, report?, reason?,
+ * message?}` shape, same try/finally cleanup. **Key diff**: Weibo needs
+ * an extra fetchUid roundtrip after cookies extraction (cookie alone
+ * doesn't carry UID — Bilibili has DedeUserID inline).
+ */
+
+const { WeiboApiClient } = require("./api-client");
+const {
+  buildSnapshot,
+  writeSnapshotJson,
+  cleanupSnapshotJson,
+} = require("./snapshot-builder");
+
+/**
+ * Pull cookies → fetchUid → 3 endpoints → write snapshot. Returns the
+ * staging path + counts + diagnostic.
+ *
+ * Throws (with typed-reason BILIBILI_-style prefix) on cookie failures.
+ * Returns with empty events on /api/config failure or any endpoint
+ * failure (partial-result tolerated — lastErrorCode surfaces the cause
+ * for UI).
+ */
+async function collect(bridge, opts = {}) {
+  if (!bridge || typeof bridge.invoke !== "function") {
+    throw new TypeError(
+      "WeiboAdbCollector.collect: bridge must expose invoke(method, params)",
+    );
+  }
+  const now = opts.now || Date.now;
+  const client = opts.apiClient || new WeiboApiClient();
+  const limits = opts.limits || {};
+
+  // 1. Pull cookies via Phase 3a extension.
+  const cookieResult = await bridge.invoke("weibo.cookies");
+  if (!cookieResult || typeof cookieResult.cookie !== "string") {
+    throw new Error(
+      "WeiboAdbCollector.collect: bridge.invoke('weibo.cookies') returned malformed payload — got cookie=" +
+        typeof cookieResult?.cookie,
+    );
+  }
+  const { cookie, diagnostic: cookieDiagnostic } = cookieResult;
+
+  // 2. fetchUid — required first call. Weibo cookie has no inline UID.
+  const uid = await client.fetchUid(cookie);
+  if (!uid) {
+    // /api/config returned login=false or non-2xx. Could be:
+    //  - cookie expired (most common — user logged out on phone)
+    //  - anti-bot 30x to login HTML (UA missing — but we set browser UA)
+    //  - IO error
+    // Surface as ExtractFailed via the hub-level wrapper; here we
+    // produce an empty-event snapshot so the registry call doesn't
+    // throw (consumers can read douyin.lastErrorCode to disambiguate).
+    const snapshot = buildSnapshot({
+      uid: 1, // sentinel — buildSnapshot requires positive; sync emits 0 events
+      displayName: opts.displayName,
+      snapshottedAt: now(),
+    });
+    const snapshotPath = writeSnapshotJson(snapshot, { dir: opts.stagingDir });
+    return {
+      snapshotPath,
+      uid: null,
+      eventCounts: { post: 0, favourite: 0, follow: 0, total: 0 },
+      lastErrorCode: client.lastErrorCode,
+      lastErrorMessage: client.lastErrorMessage,
+      cookieDiagnostic: cookieDiagnostic || null,
+      uidFetchFailed: true,
+    };
+  }
+
+  // 3. Parallel fetch — partial failure tolerated (client returns []).
+  const [posts, favourites, follows] = await Promise.all([
+    client.fetchPosts(cookie, uid, {
+      limit: Number.isInteger(limits.post) ? limits.post : undefined,
+    }),
+    client.fetchFavourites(cookie, {
+      limit: Number.isInteger(limits.favourite) ? limits.favourite : undefined,
+    }),
+    client.fetchFollows(cookie, uid, {
+      limit: Number.isInteger(limits.follow) ? limits.follow : undefined,
+    }),
+  ]);
+
+  // 4. Build snapshot + write.
+  const snapshot = buildSnapshot({
+    uid,
+    displayName: opts.displayName,
+    posts,
+    favourites,
+    follows,
+    snapshottedAt: now(),
+  });
+  const snapshotPath = writeSnapshotJson(snapshot, { dir: opts.stagingDir });
+
+  return {
+    snapshotPath,
+    uid,
+    eventCounts: {
+      post: posts.length,
+      favourite: favourites.length,
+      follow: follows.length,
+      total: snapshot.events.length,
+    },
+    lastErrorCode: client.lastErrorCode,
+    lastErrorMessage: client.lastErrorMessage,
+    cookieDiagnostic: cookieDiagnostic || null,
+    uidFetchFailed: false,
+  };
+}
+
+/**
+ * Convenience: collect + registry.syncAdapter("social-weibo") + cleanup.
+ */
+async function collectAndSync(bridge, registry, opts = {}) {
+  if (!registry || typeof registry.syncAdapter !== "function") {
+    throw new TypeError(
+      "WeiboAdbCollector.collectAndSync: registry must expose syncAdapter(name, options)",
+    );
+  }
+  const collectResult = await collect(bridge, opts);
+  let syncReport = null;
+  let cleanupFailed = false;
+  try {
+    syncReport = await registry.syncAdapter("social-weibo", {
+      inputPath: collectResult.snapshotPath,
+    });
+  } finally {
+    try {
+      cleanupSnapshotJson(collectResult.snapshotPath);
+    } catch (_e) {
+      cleanupFailed = true;
+    }
+  }
+  return {
+    ...syncReport,
+    weibo: {
+      uid: collectResult.uid,
+      eventCounts: collectResult.eventCounts,
+      lastErrorCode: collectResult.lastErrorCode,
+      lastErrorMessage: collectResult.lastErrorMessage,
+      cookieDiagnostic: collectResult.cookieDiagnostic,
+      uidFetchFailed: collectResult.uidFetchFailed,
+      cleanupFailed,
+    },
+  };
+}
+
+module.exports = {
+  collect,
+  collectAndSync,
+};