@chainlesschain/personal-data-hub 0.3.6 → 0.3.7

package/lib/adapters/social-toutiao-adb/cookies-extension.js

@@ -0,0 +1,266 @@
+"use strict";
+
+/**
+ * Phase 6c (Toutiao C 路径 — 2026-05-25): toutiao.cookies ADB extension factory.
+ *
+ * Mirror of `social-xiaohongshu-adb/cookies-extension.js` (P3c). Reads the
+ * Chromium cookies SQLite from Toutiao Android app (com.ss.android.article.news)
+ * via `su -c base64`. Returns Cookie header + pre-extracted uid (if present
+ * in cookie) + a session-id liveness flag.
+ *
+ * Required cookies (without these, fetchProfile returns status_code != 0):
+ *  - one of `passport_uid` / `multi_sids` / `__ac_uid` → uid identification
+ *  - `sessionid` OR `sessionid_ss` → passport endpoint auth
+ *
+ * Either missing → TOUTIAO_COOKIES_INCOMPLETE so UI surfaces a "relog on
+ * phone" banner. _signature endpoint cookies (msToken / __ac_nonce / ttwid)
+ * rotate fast (5-15min) so we don't fail-fast on them — fetchProfile will
+ * reveal whether they're stale.
+ *
+ * Returns:
+ *   {
+ *     cookie: string,
+ *     uid: string|null,            // best-effort uid pre-extract (may be null
+ *                                   // if cookie only has session — caller
+ *                                   // falls back to api.fetchProfile)
+ *     extractedAt: number,
+ *     diagnostic: {
+ *       cookieCount: number,
+ *       hadEncrypted: boolean,
+ *       cookieNames: string[],
+ *     }
+ *   }
+ */
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+const crypto = require("node:crypto");
+
+const {
+  readChromiumCookies,
+} = require("../social-bilibili-adb/chromium-cookies-reader");
+
+const TOUTIAO_COOKIES_REMOTE_PATH =
+  "/data/data/com.ss.android.article.news/app_webview/Default/Cookies";
+
+const TOUTIAO_COOKIE_HOST_DOMAIN = "toutiao.com";
+
+/**
+ * Lenient — at least one of sessionid / sessionid_ss must be present, AND
+ * at least one of the uid candidates. Toutiao's anti-bot cookies (msToken /
+ * __ac_nonce / ttwid) rotate fast so we don't fail-fast on them.
+ */
+const TOUTIAO_SESSION_COOKIES = Object.freeze(["sessionid", "sessionid_ss"]);
+const TOUTIAO_UID_COOKIES = Object.freeze([
+  "passport_uid",
+  "multi_sids",
+  "__ac_uid",
+  "tt_uid",
+]);
+
+async function pullCookiesViaSu(adb, serial, opts) {
+  const adbOpts = { serial, timeoutMs: opts?.timeoutMs || 60_000 };
+  const lsOut = await adb(
+    [
+      "shell",
+      "su",
+      "-c",
+      `ls ${TOUTIAO_COOKIES_REMOTE_PATH} 2>/dev/null || echo NOT_FOUND`,
+    ],
+    adbOpts,
+  );
+  const lsLine = lsOut.replace(/\r+$/gm, "").trim();
+  if (lsLine === "NOT_FOUND" || lsLine === "") {
+    throw new Error(
+      "TOUTIAO_NOT_INSTALLED: " +
+        TOUTIAO_COOKIES_REMOTE_PATH +
+        " not found. Install Toutiao App (今日头条 com.ss.android.article.news) + log in once, then retry. Note: 极速版 (.lite) uses a different package — only the standard app is supported.",
+    );
+  }
+  const idOut = await adb(["shell", "su", "-c", "id -u"], adbOpts);
+  const idLine = idOut.replace(/\r+$/gm, "").trim();
+  if (idLine !== "0" && !idLine.includes("uid=0")) {
+    throw new Error(
+      "TOUTIAO_NO_ROOT: this phone isn't rooted (su returned `" +
+        idLine.substring(0, 60) +
+        "`). Toutiao release APK isn't debuggable, so root is required to read the Chromium cookies DB.",
+    );
+  }
+  const b64 = await adb(
+    [
+      "shell",
+      "su",
+      "-c",
+      `base64 ${TOUTIAO_COOKIES_REMOTE_PATH} | tr -d '\\n\\r'`,
+    ],
+    { ...adbOpts, timeoutMs: opts?.timeoutMs || 60_000 },
+  );
+  const b64Clean = b64.replace(/[\r\n\t ]+/g, "");
+  if (b64Clean.length === 0) {
+    throw new Error(
+      "TOUTIAO_COOKIES_EMPTY: base64 stream returned 0 bytes (su exec may have silently failed on MIUI / OEM ROM)",
+    );
+  }
+  let buf;
+  try {
+    buf = Buffer.from(b64Clean, "base64");
+  } catch (e) {
+    throw new Error(
+      "TOUTIAO_BASE64_PARSE: stream wasn't valid base64 (" +
+        (e.message || String(e)) +
+        ")",
+    );
+  }
+  if (buf.length < 1024) {
+    throw new Error(
+      "TOUTIAO_COOKIES_TRUNCATED: decoded file is only " +
+        buf.length +
+        " bytes — expected ≥4KB sqlite",
+    );
+  }
+  const magic = buf.subarray(0, 16).toString("latin1");
+  if (!magic.startsWith("SQLite format 3")) {
+    throw new Error(
+      "TOUTIAO_NOT_SQLITE: decoded file lacks `SQLite format 3` magic header",
+    );
+  }
+  const tmpDir = os.tmpdir();
+  const tmpFile = path.join(
+    tmpDir,
+    `cc-toutiao-cookies-${crypto.randomUUID()}.db`,
+  );
+  fs.writeFileSync(tmpFile, buf);
+  return tmpFile;
+}
+
+/**
+ * Build a Cookie header from a chromium-cookies array. Picks the
+ * longest hostKey when multiple entries share a name (mirror of Xhs +
+ * Weibo logic — handles .toutiao.com vs www.toutiao.com duplicates).
+ *
+ * Extracts uid best-effort: passport_uid > multi_sids first segment >
+ * __ac_uid > tt_uid. Returns null uid if no candidate present.
+ */
+function assembleToutiaoCookieHeader(cookies) {
+  if (!Array.isArray(cookies)) {
+    throw new TypeError(
+      "assembleToutiaoCookieHeader: cookies must be an array",
+    );
+  }
+  const byName = new Map();
+  for (const c of cookies) {
+    if (
+      !byName.has(c.name) ||
+      c.hostKey.length > (byName.get(c.name).hostKey || "").length
+    ) {
+      byName.set(c.name, c);
+    }
+  }
+  const hasSession = TOUTIAO_SESSION_COOKIES.some((n) => byName.has(n));
+  const uid = pickUidFromCookieMap(byName);
+  const present = new Set(byName.keys());
+  if (!hasSession) {
+    return {
+      header: null,
+      uid: null,
+      present,
+      missing: TOUTIAO_SESSION_COOKIES.filter((n) => !byName.has(n)),
+    };
+  }
+  const header = Array.from(byName.values())
+    .map((c) => `${c.name}=${c.value}`)
+    .join("; ");
+  return { header, uid, present, missing: [] };
+}
+
+function pickUidFromCookieMap(byName) {
+  const passport = byName.get("passport_uid")?.value;
+  if (passport && /^\d+$/.test(passport) && passport !== "0") {
+    return passport;
+  }
+  const multi = byName.get("multi_sids")?.value;
+  if (multi) {
+    // Format "12345:abcd;67890:efgh" — pick first uid segment
+    const firstUid = multi.split(";")[0].split(":")[0].trim();
+    if (firstUid && /^\d+$/.test(firstUid) && firstUid !== "0") {
+      return firstUid;
+    }
+  }
+  const acUid = byName.get("__ac_uid")?.value;
+  if (acUid && /^\d+$/.test(acUid) && acUid !== "0") {
+    return acUid;
+  }
+  const ttUid = byName.get("tt_uid")?.value;
+  if (ttUid && /^\d+$/.test(ttUid) && ttUid !== "0") {
+    return ttUid;
+  }
+  return null;
+}
+
+function createToutiaoCookiesExtension(factoryOpts = {}) {
+  const timeoutMs = factoryOpts.timeoutMs || 60_000;
+  const onCleanupFailed = factoryOpts.onCleanupFailed || (() => {});
+
+  return async function toutiaoCookiesHandler(_params, ctx) {
+    if (
+      !ctx ||
+      typeof ctx.adb !== "function" ||
+      typeof ctx.pickDevice !== "function"
+    ) {
+      throw new TypeError(
+        "toutiao.cookies extension: ctx must provide {adb, pickDevice}",
+      );
+    }
+    const serial = await ctx.pickDevice();
+    let tmpFile = null;
+    try {
+      tmpFile = await pullCookiesViaSu(ctx.adb, serial, { timeoutMs });
+      const cookies = readChromiumCookies(tmpFile, TOUTIAO_COOKIE_HOST_DOMAIN);
+      const cookieCount = cookies.length;
+      const hadEncrypted = (cookies._skippedEncryptedCount || 0) > 0;
+      const { header, uid, missing, present } =
+        assembleToutiaoCookieHeader(cookies);
+      if (header === null) {
+        throw new Error(
+          "TOUTIAO_COOKIES_INCOMPLETE: missing required session cookies " +
+            JSON.stringify(missing) +
+            ". Likely the user logged out, or has never logged in via the Toutiao app's WebView (open any article link to populate). hadEncrypted=" +
+            hadEncrypted +
+            ".",
+        );
+      }
+      return {
+        cookie: header,
+        uid,
+        extractedAt: Date.now(),
+        diagnostic: {
+          cookieCount,
+          hadEncrypted,
+          cookieNames: Array.from(present),
+        },
+      };
+    } finally {
+      if (tmpFile) {
+        try {
+          fs.unlinkSync(tmpFile);
+        } catch (_e) {
+          onCleanupFailed(tmpFile);
+        }
+      }
+    }
+  };
+}
+
+module.exports = {
+  createToutiaoCookiesExtension,
+  TOUTIAO_COOKIES_REMOTE_PATH,
+  TOUTIAO_COOKIE_HOST_DOMAIN,
+  TOUTIAO_SESSION_COOKIES,
+  TOUTIAO_UID_COOKIES,
+  assembleToutiaoCookieHeader,
+  _internals: {
+    pullCookiesViaSu,
+    pickUidFromCookieMap,
+  },
+};

package/lib/adapters/social-toutiao-adb/index.js

@@ -0,0 +1,52 @@
+"use strict";
+
+/**
+ * social-toutiao-adb — Phase 6c (Toutiao C 路径) entry.
+ *
+ * Desktop ADB pulls Chromium cookies from Toutiao Android app
+ * (com.ss.android.article.news) via `su -c base64`, then runs Toutiao web
+ * HTTP via ToutiaoApiClient. _signature path: desktop wiring injects
+ * ToutiaoSignBridge (Electron WebContentsView running acrawler.js, ~100%
+ * hit rate); CLI / test contexts get NullSignProvider → signed endpoints
+ * short-circuit with lastErrorCode=-99.
+ *
+ * Pipeline:
+ *   bridge.invoke("toutiao.cookies") → {cookie, uid}
+ *     → ToutiaoApiClient.fetchProfile (no _sig)
+ *     → fetchFeed + fetchCollection + fetchSearchHistory (signed, parallel)
+ *     → buildSnapshot + writeSnapshotJson
+ *     → registry.syncAdapter("social-toutiao", { inputPath })
+ */
+
+const {
+  createToutiaoCookiesExtension,
+  TOUTIAO_COOKIES_REMOTE_PATH,
+  TOUTIAO_COOKIE_HOST_DOMAIN,
+  TOUTIAO_SESSION_COOKIES,
+  TOUTIAO_UID_COOKIES,
+  assembleToutiaoCookieHeader,
+} = require("./cookies-extension");
+const { ToutiaoApiClient } = require("./api-client");
+const {
+  buildSnapshot,
+  writeSnapshotJson,
+  cleanupSnapshotJson,
+  SNAPSHOT_SCHEMA_VERSION,
+} = require("./snapshot-builder");
+const { collect, collectAndSync } = require("./collector");
+
+module.exports = {
+  createToutiaoCookiesExtension,
+  TOUTIAO_COOKIES_REMOTE_PATH,
+  TOUTIAO_COOKIE_HOST_DOMAIN,
+  TOUTIAO_SESSION_COOKIES,
+  TOUTIAO_UID_COOKIES,
+  assembleToutiaoCookieHeader,
+  ToutiaoApiClient,
+  buildSnapshot,
+  writeSnapshotJson,
+  cleanupSnapshotJson,
+  SNAPSHOT_SCHEMA_VERSION,
+  collect,
+  collectAndSync,
+};

package/lib/adapters/social-toutiao-adb/snapshot-builder.js

@@ -0,0 +1,148 @@
+"use strict";
+
+/**
+ * Phase 6c (Toutiao C 路径 — 2026-05-25): API responses → snapshot JSON.
+ *
+ * Matches the existing `social-toutiao` adapter's snapshot mode schema
+ * (schemaVersion=1). Kinds: profile / read / collection / search.
+ *
+ * Toutiao uid is numeric string; account.uid is set verbatim.
+ */
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+const crypto = require("node:crypto");
+
+const SNAPSHOT_SCHEMA_VERSION = 1;
+
+function buildSnapshot(input) {
+  if (!input || typeof input !== "object") {
+    throw new TypeError("buildSnapshot: input must be an object");
+  }
+  const uid = input.uid;
+  if (typeof uid !== "string" || uid.length === 0) {
+    throw new TypeError("buildSnapshot: input.uid must be a non-empty string");
+  }
+  const snapshottedAt =
+    Number.isFinite(input.snapshottedAt) && input.snapshottedAt > 0
+      ? input.snapshottedAt
+      : Date.now();
+  const account = {
+    uid,
+    displayName: typeof input.displayName === "string" ? input.displayName : "",
+  };
+  const events = [];
+
+  // profile (1 event — matches Android collector emitting one profile event
+  // per snapshot — adapter normalize() upserts the person record).
+  if (input.profile && typeof input.profile === "object") {
+    const p = input.profile;
+    events.push({
+      kind: "profile",
+      id: `profile-${uid}`,
+      capturedAt: snapshottedAt,
+      uid,
+      nickname: p.nickname || account.displayName,
+      avatarUrl: p.avatarUrl || null,
+      mobile: p.mobile || null,
+      description: p.description || null,
+      followingCount:
+        typeof p.followingCount === "number" ? p.followingCount : 0,
+      followerCount:
+        typeof p.followerCount === "number" ? p.followerCount : 0,
+      mediaId: p.mediaId || null,
+    });
+  }
+
+  // read history
+  const feed = Array.isArray(input.feed) ? input.feed : [];
+  feed.forEach((f, idx) => {
+    if (!f || typeof f !== "object") return;
+    events.push({
+      kind: "read",
+      id: f.itemId ? `read-${f.itemId}` : `read-${idx}`,
+      capturedAt:
+        typeof f.publishedAt === "number" && f.publishedAt > 0
+          ? f.publishedAt
+          : snapshottedAt,
+      itemId: f.itemId || null,
+      title: f.title || null,
+      category: f.category || null,
+      author: f.author || null,
+      readDuration: typeof f.readDuration === "number" ? f.readDuration : 0,
+      source: f.source || null,
+    });
+  });
+
+  // collection
+  const collection = Array.isArray(input.collection) ? input.collection : [];
+  collection.forEach((c, idx) => {
+    if (!c || typeof c !== "object") return;
+    events.push({
+      kind: "collection",
+      id: c.itemId ? `collect-${c.itemId}` : `collect-${idx}`,
+      capturedAt:
+        typeof c.savedAt === "number" && c.savedAt > 0
+          ? c.savedAt
+          : snapshottedAt,
+      itemId: c.itemId || null,
+      title: c.title || null,
+      category: c.category || null,
+      author: c.author || null,
+    });
+  });
+
+  // search history
+  const search = Array.isArray(input.search) ? input.search : [];
+  search.forEach((s, idx) => {
+    if (!s || typeof s !== "object" || !s.keyword) return;
+    events.push({
+      kind: "search",
+      id: `search-${s.keyword}:${s.searchedAt || idx}`,
+      capturedAt:
+        typeof s.searchedAt === "number" && s.searchedAt > 0
+          ? s.searchedAt
+          : snapshottedAt,
+      keyword: s.keyword,
+      searchAt: s.searchedAt || snapshottedAt,
+    });
+  });
+
+  return {
+    schemaVersion: SNAPSHOT_SCHEMA_VERSION,
+    snapshottedAt,
+    account,
+    events,
+  };
+}
+
+function writeSnapshotJson(snapshot, opts = {}) {
+  const dir = opts.dir || os.tmpdir();
+  const fileName =
+    opts.fileName || `cc-toutiao-snapshot-${crypto.randomUUID()}.json`;
+  if (fileName.includes("/") || fileName.includes("\\")) {
+    throw new Error(
+      "writeSnapshotJson: opts.fileName must be a basename, not a path",
+    );
+  }
+  const full = path.join(dir, fileName);
+  fs.writeFileSync(full, JSON.stringify(snapshot), "utf-8");
+  return full;
+}
+
+function cleanupSnapshotJson(filePath) {
+  if (!filePath) return;
+  try {
+    fs.unlinkSync(filePath);
+  } catch (_e) {
+    // ignore
+  }
+}
+
+module.exports = {
+  buildSnapshot,
+  writeSnapshotJson,
+  cleanupSnapshotJson,
+  SNAPSHOT_SCHEMA_VERSION,
+};

package/lib/adapters/social-xiaohongshu-adb/api-client.js

@@ -19,6 +19,7 @@
  */
 
 const { computeXsXt } = require("./sign");
+const { NULL_SIGN_PROVIDER } = require("../../sign-providers");
 
 const DEFAULT_BASE_URL = "https://edith.xiaohongshu.com/";
 
@@ -80,19 +81,49 @@ class XhsApiClient {
       );
     }
     this._now = opts.now || Date.now;
+    // Phase 6b: signProvider injectable. Desktop wiring injects
+    // XhsSignBridge (Electron WebContentsView running xhs.js, ~100% hit
+    // rate). CLI / tests get NULL_SIGN_PROVIDER → falls back to the
+    // in-process best-effort computeXsXt (~60% GET / <30% POST hit).
+    // Both code paths are present so the client works in either context
+    // without the caller having to swap api-client implementations.
+    this.signProvider = opts.signProvider || NULL_SIGN_PROVIDER;
     this.lastErrorCode = 0;
     this.lastErrorMessage = null;
+    // Diagnostic counters — collector reads these to decide whether to
+    // surface "bridge upgrade succeeded" in the report.
+    this._bridgeHits = 0;
+    this._fallbackHits = 0;
   }
 
   async _doGetJson(url, cookie, a1, requireSign) {
     const headers = { ...BROWSER_HEADERS, Cookie: cookie };
     if (requireSign && a1) {
       const pathWithQuery = url.pathname + url.search;
-      const { xs, xt } = computeXsXt(pathWithQuery, null, a1, {
-        now: this._now,
-      });
-      headers["X-S"] = xs;
-      headers["X-T"] = xt;
+      // Phase 6b: prefer bridge over in-process computeXsXt.
+      // signedHeaders is async — bridge does executeJavaScript across
+      // Electron IPC. Returns {} on cold bridge / xhs.js rotation / IPC
+      // error, in which case we fall back to the best-effort md5.
+      const bridgeHeaders = await this.signProvider.signedHeaders(
+        url,
+        `${pathWithQuery}|`,
+      );
+      const bridgeKeys = Object.keys(bridgeHeaders);
+      if (bridgeKeys.length > 0) {
+        // Bridge produced headers — use them verbatim. xhs.js returns
+        // X-s / X-t (lowercase t in some builds) / X-s-common; we let
+        // the bridge's normalizeXhsHeader handle case.
+        Object.assign(headers, bridgeHeaders);
+        this._bridgeHits += 1;
+      } else {
+        // Fallback: in-process best-effort md5 (P3c path).
+        const { xs, xt } = computeXsXt(pathWithQuery, null, a1, {
+          now: this._now,
+        });
+        headers["X-S"] = xs;
+        headers["X-T"] = xt;
+        this._fallbackHits += 1;
+      }
     }
     try {
       const resp = await this._fetch(url.toString(), {