@chainlesschain/personal-data-hub 0.4.29 → 0.4.31

package/__tests__/adapters/social-toutiao-adb-article.test.js

@@ -1,155 +0,0 @@
-/**
- * Toutiao article reader tests (real-device-driven 2026-06-18: the user's
- * exported news_article.db `article` table = 48 feed-cache rows; title lives in
- * the share_info JSON blob, not a column).
- *
- * Two layers: pure parsing via injected fake Database, + a real better-sqlite3
- * db + real LocalVault round-trip proving the hand-built BROWSE events pass
- * schema validation, are searchable, and re-ingest dedups on the stable
- * originalId.
- */
-"use strict";
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-
-const fs = require("node:fs");
-const path = require("node:path");
-const os = require("node:os");
-
-const { LocalVault } = require("../../lib/vault");
-const { generateKeyHex } = require("../../lib/key-providers");
-const {
-  ARTICLE_TABLE,
-  readToutiaoArticles,
-  buildArticleEvents,
-  articlesToVault,
-  _internals,
-} = require("../../lib/adapters/social-toutiao-adb/article-reader");
-
-function makeFakeDb(rows, { table = ARTICLE_TABLE } = {}) {
-  const cols = [
-    "group_id", "item_id", "share_info", "ext_json", "share_url",
-    "behot_time", "read_timestamp", "is_user_digg", "is_user_repin",
-  ];
-  return class FakeDb {
-    constructor() {}
-    prepare(sql) {
-      return {
-        get: (arg) => (/sqlite_master/.test(sql) ? (arg === table ? { name: table } : undefined) : undefined),
-        all: () => {
-          if (/table_info/.test(sql)) return cols.map((name) => ({ name }));
-          if (/FROM "/.test(sql)) return rows;
-          return [];
-        },
-      };
-    }
-    close() {}
-  };
-}
-
-describe("readToutiaoArticles (injected fake db)", () => {
-  it("parses title from share_info, strips the brand suffix, drops url tracking query", () => {
-    const Db = makeFakeDb([
-      {
-        group_id: 100, behot_time: 1781700000, read_timestamp: 0, is_user_digg: 1, is_user_repin: 0,
-        share_info: JSON.stringify({ title: "5月汽车出口延续快速增长态势 - 今日头条", share_url: "https://m.toutiao.com/g/100/?app=x&category_new=headline" }),
-        share_url: "https://m.toutiao.com/g/100/?category_new=headline",
-      },
-    ]);
-    const { articles } = readToutiaoArticles("x.db", { _databaseClass: Db });
-    expect(articles).toHaveLength(1);
-    expect(articles[0].title).toBe("5月汽车出口延续快速增长态势"); // suffix stripped
-    expect(articles[0].url).toBe("https://m.toutiao.com/g/100/"); // query dropped
-    expect(articles[0].category).toBe("headline");
-    expect(articles[0].digg).toBe(true);
-    expect(articles[0].behotTime).toBe(1781700000 * 1000);
-  });
-
-  it("falls back to ext_json.title when share_info has none, and skips untitled rows", () => {
-    const Db = makeFakeDb([
-      { group_id: 1, ext_json: JSON.stringify({ title: "来自 ext_json 的标题" }), share_info: "{}" },
-      { group_id: 2, share_info: "{}", ext_json: "{}" }, // untitled → dropped
-    ]);
-    const { articles } = readToutiaoArticles("x.db", { _databaseClass: Db });
-    expect(articles).toHaveLength(1);
-    expect(articles[0].title).toBe("来自 ext_json 的标题");
-  });
-
-  it("returns no articles when the table is absent", () => {
-    const Db = makeFakeDb([], { table: "other" });
-    expect(readToutiaoArticles("x.db", { _databaseClass: Db }).articles).toEqual([]);
-  });
-
-  it("buildArticleEvents → BROWSE events, social-toutiao source, stable originalId, read flag", () => {
-    const { events } = buildArticleEvents(
-      [{ groupId: "55", title: "标题", url: "u", category: "headline", behotTime: 2, readTimestamp: 1781700000000, digg: true, repin: false }],
-      { now: 1781800000000 },
-    );
-    expect(events).toHaveLength(1);
-    const e = events[0];
-    expect(e.subtype).toBe("browse");
-    expect(e.source.adapter).toBe("social-toutiao");
-    expect(e.source.originalId).toBe("social-toutiao:article:55");
-    expect(e.occurredAt).toBe(1781700000000); // read_timestamp wins over behot
-    expect(e.extra.kind).toBe("article");
-    expect(e.extra.read).toBe(true);
-    expect(e.extra.digg).toBe(true);
-  });
-
-  it("extractCategory / extractUrl helpers", () => {
-    expect(_internals.extractCategory({ share_url: "x?a=1&category_new=my_tabs_digg&b=2" })).toBe("my_tabs_digg");
-    expect(_internals.extractUrl({ share_info: JSON.stringify({ share_url: "https://h/g/1/?t=1" }) })).toBe("https://h/g/1/");
-  });
-});
-
-describe("articlesToVault — real sqlite + real vault", () => {
-  let dir, dbPath, vdir, vault;
-
-  beforeAll(() => {
-    const Database = require("better-sqlite3-multiple-ciphers");
-    dir = fs.mkdtempSync(path.join(os.tmpdir(), "tt-article-"));
-    dbPath = path.join(dir, "news_article.db");
-    const db = new Database(dbPath);
-    db.exec(
-      "CREATE TABLE article (group_id INTEGER, item_id INTEGER, share_info TEXT, ext_json TEXT, " +
-        "share_url TEXT, behot_time INTEGER, read_timestamp INTEGER, is_user_digg INTEGER, is_user_repin INTEGER)",
-    );
-    const ins = db.prepare(
-      "INSERT INTO article (group_id, share_info, share_url, behot_time, read_timestamp, is_user_digg, is_user_repin) VALUES (?,?,?,?,?,?,?)",
-    );
-    ins.run(101, JSON.stringify({ title: "新华视点丨三峡水运新通道 - 今日头条", share_url: "https://m.toutiao.com/g/101/?x=1&category_new=headline" }), "https://m.toutiao.com/g/101/?category_new=headline", 1781700000, 0, 0, 0);
-    ins.run(102, JSON.stringify({ title: "5月汽车出口延续快速增长态势 - 今日头条", share_url: "https://m.toutiao.com/g/102/" }), "https://m.toutiao.com/g/102/?category_new=my_tabs_digg", 1781700100, 1781700200, 1, 0);
-    ins.run(103, "{}", "https://m.toutiao.com/g/103/", 1781700300, 0, 0, 0); // untitled → not ingested
-    db.close();
-
-    vdir = fs.mkdtempSync(path.join(os.tmpdir(), "tt-article-vault-"));
-    vault = new LocalVault({ path: path.join(vdir, "v.db"), key: generateKeyHex() });
-    vault.open();
-  });
-
-  afterAll(() => {
-    try { vault.close(); } catch (_e) { /* best-effort */ }
-    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
-    try { fs.rmSync(vdir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
-  });
-
-  it("reads titled articles and ingests valid BROWSE events", () => {
-    const r = articlesToVault(vault, dbPath, { now: 1781900000000 });
-    expect(r.articles).toBe(2); // the untitled row is skipped
-    expect(r.ingested).toBe(2); // both passed schema validation
-    expect(r.digg).toBe(1);
-    expect(r.read).toBe(1);
-
-    const events = vault.queryEvents({ limit: 100 }) || [];
-    const mine = events.filter((e) => e.extra && e.extra.kind === "article");
-    expect(mine.length).toBe(2);
-    expect(mine.every((e) => e.source.adapter === "social-toutiao")).toBe(true);
-  });
-
-  it("re-ingest dedups on the stable per-article originalId", () => {
-    articlesToVault(vault, dbPath, { now: 1781999999999 });
-    const events = vault.queryEvents({ limit: 100 }) || [];
-    const mine = events.filter((e) => e.extra && e.extra.kind === "article");
-    expect(mine.length).toBe(2); // still two — updated, not duplicated
-  });
-});

package/__tests__/adapters/social-toutiao-adb-collector.test.js

@@ -1,378 +0,0 @@
-"use strict";
-
-import { describe, it, expect, vi } from "vitest";
-const os = require("node:os");
-
-const {
-  collect,
-  collectAndSync,
-} = require("../../lib/adapters/social-toutiao-adb/collector");
-const {
-  ToutiaoApiClient,
-} = require("../../lib/adapters/social-toutiao-adb/api-client");
-
-function makeFakeFetch(responses) {
-  const calls = [];
-  const fakeFetch = async (urlStr, opts) => {
-    calls.push({ url: urlStr, opts });
-    for (const [pattern, payload] of responses) {
-      if (urlStr.includes(pattern)) {
-        const resolved =
-          typeof payload === "function" ? await payload(urlStr, opts) : payload;
-        return {
-          ok: resolved.status == null || resolved.status === 200,
-          status: resolved.status || 200,
-          text: async () => resolved.body,
-        };
-      }
-    }
-    throw new Error("fake fetch: no response for " + urlStr);
-  };
-  return { fakeFetch, calls };
-}
-
-const HAPPY_RESPONSES = [
-  [
-    "passport/account/info/v2",
-    {
-      body: JSON.stringify({
-        status_code: 0,
-        data: {
-          user_id: "12345",
-          screen_name: "Alice",
-          avatar_url: "https://a/x.jpg",
-        },
-      }),
-    },
-  ],
-  [
-    "api/news/feed/v90",
-    {
-      body: JSON.stringify({
-        data: [{ group_id: "G1", title: "T1", behot_time: 1700000000 }],
-      }),
-    },
-  ],
-  [
-    "article/v2/tab_comments",
-    {
-      body: JSON.stringify({
-        data: [{ group_id: "C1", title: "Saved", behot_time: 1700001000 }],
-      }),
-    },
-  ],
-  [
-    "api/search/content",
-    {
-      body: JSON.stringify({
-        data: { user_search_history: [{ keyword: "kw", time: 1700002000 }] },
-      }),
-    },
-  ],
-];
-
-function makeBridge(invokeResult, accountResult) {
-  return {
-    invoke: vi.fn(async (method) => {
-      if (method === "toutiao.account") {
-        // Mirror real wiring: a separate extension. Tests that don't wire it
-        // get a throw (collector falls through gracefully).
-        if (accountResult === undefined) {
-          throw new Error("toutiao.account not wired in this test");
-        }
-        return accountResult;
-      }
-      return invokeResult;
-    }),
-  };
-}
-
-const COOKIE_PAYLOAD = {
-  cookie: "sessionid=abc; passport_uid=12345",
-  uid: "12345",
-  diagnostic: { cookieCount: 2, hadEncrypted: false, cookieNames: ["sessionid", "passport_uid"] },
-};
-
-describe("collect — happy path with signProvider", () => {
-  it("warmUp → signed endpoints → shutdown", async () => {
-    const { fakeFetch } = makeFakeFetch(HAPPY_RESPONSES);
-    const calls = [];
-    const sign = {
-      warmUp: vi.fn(async (c) => calls.push({ warmUp: c })),
-      signUrl: vi.fn(async (url) => {
-        const u = new URL(String(url));
-        u.searchParams.set("_signature", "BRIDGE_SIG");
-        return u;
-      }),
-      shutdown: vi.fn(async () => calls.push("shutdown")),
-    };
-    const client = new ToutiaoApiClient({
-      fetch: fakeFetch,
-      signProvider: sign,
-    });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      signProvider: sign,
-      stagingDir: os.tmpdir(),
-    });
-    expect(sign.warmUp).toHaveBeenCalledWith(COOKIE_PAYLOAD.cookie);
-    expect(sign.shutdown).toHaveBeenCalledOnce();
-    expect(r.uid).toBe("12345");
-    expect(r.nickname).toBe("Alice");
-    expect(r.profileFetchFailed).toBe(false);
-    expect(r.eventCounts.feed).toBe(1);
-    expect(r.eventCounts.collection).toBe(1);
-    expect(r.eventCounts.search).toBe(1);
-    expect(r.eventCounts.profile).toBe(1);
-    expect(r.signProviderHits).toBe(3); // 3 signed endpoints
-    expect(r.signProviderFallbacks).toBe(0);
-  });
-});
-
-describe("collect — fallback path (no signProvider)", () => {
-  it("3 signed endpoints short-circuit; profile still emitted", async () => {
-    const { fakeFetch } = makeFakeFetch(HAPPY_RESPONSES);
-    const client = new ToutiaoApiClient({ fetch: fakeFetch });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      stagingDir: os.tmpdir(),
-    });
-    expect(r.uid).toBe("12345");
-    expect(r.profileFetchFailed).toBe(false);
-    expect(r.eventCounts.profile).toBe(1);
-    expect(r.eventCounts.feed).toBe(0); // short-circuit
-    expect(r.eventCounts.collection).toBe(0);
-    expect(r.eventCounts.search).toBe(0);
-    expect(r.signProviderUsed).toBe("none");
-    expect(r.signProviderHits).toBe(0);
-    expect(r.signProviderFallbacks).toBe(3);
-  });
-});
-
-describe("collect — profile fetch fails", () => {
-  it("emits empty snapshot with cookie-derived uid + profileFetchFailed=true", async () => {
-    const { fakeFetch } = makeFakeFetch([
-      [
-        "passport/account/info/v2",
-        {
-          body: JSON.stringify({
-            status_code: 1,
-            status_msg: "token expired",
-          }),
-        },
-      ],
-    ]);
-    const client = new ToutiaoApiClient({ fetch: fakeFetch });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      stagingDir: os.tmpdir(),
-    });
-    expect(r.profileFetchFailed).toBe(true);
-    expect(r.uid).toBe("12345"); // from cookie pre-extract
-    expect(r.eventCounts.total).toBe(0);
-    expect(r.lastErrorCode).toBe(1);
-  });
-
-  it("falls back to 'unknown-user' uid when cookie pre-extract also empty", async () => {
-    const { fakeFetch } = makeFakeFetch([
-      [
-        "passport/account/info/v2",
-        { body: JSON.stringify({ status_code: 1, status_msg: "expired" }) },
-      ],
-    ]);
-    const client = new ToutiaoApiClient({ fetch: fakeFetch });
-    const r = await collect(
-      makeBridge({
-        cookie: "sessionid=abc",
-        uid: null,
-        diagnostic: {},
-      }),
-      { apiClient: client, stagingDir: os.tmpdir() },
-    );
-    expect(r.uid).toBe(null);
-    expect(r.profileFetchFailed).toBe(true);
-  });
-
-  it("profile permission-denied (error_code 16) BUT cookie uid + signer → feed/collection/search still collect", async () => {
-    // Real-device 2026-06-11: logged-in Toutiao returns passport error_code 16
-    // 该应用无权限. We must NOT abort — feed is cookie-identified, so with a
-    // SignBridge the signed endpoints still flow. Profile event is skipped, but
-    // the headline error (16) is surfaced and feed/collection/search collect.
-    const { fakeFetch } = makeFakeFetch([
-      [
-        "passport/account/info/v2",
-        {
-          body: JSON.stringify({
-            message: "error",
-            data: { error_code: 16, description: "该应用无权限" },
-          }),
-        },
-      ],
-      ...HAPPY_RESPONSES.slice(1), // feed / comments / search responses
-    ]);
-    const sign = {
-      warmUp: vi.fn(async () => {}),
-      signUrl: vi.fn(async (url) => {
-        const u = new URL(String(url));
-        u.searchParams.set("_signature", "BRIDGE_SIG");
-        return u;
-      }),
-      shutdown: vi.fn(async () => {}),
-    };
-    const client = new ToutiaoApiClient({ fetch: fakeFetch, signProvider: sign });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      signProvider: sign,
-      stagingDir: os.tmpdir(),
-    });
-    expect(r.profileFetchFailed).toBe(true);
-    expect(r.uid).toBe("12345"); // cookie-derived
-    expect(r.lastErrorCode).toBe(16); // headline profile error preserved
-    expect(r.lastErrorMessage).toBe("该应用无权限");
-    expect(r.eventCounts.profile).toBe(0); // no profile event
-    expect(r.eventCounts.feed).toBe(1); // ← previously 0 (aborted before signing)
-    expect(r.eventCounts.collection).toBe(1);
-    expect(r.eventCounts.search).toBe(1);
-    expect(r.eventCounts.total).toBe(3);
-  });
-
-  it("profile error_code 16 + NO cookie uid → recovers uid from local account_db, collects signed endpoints", async () => {
-    // Real-device 2026-06-11: web profile permission-denied AND the WebView
-    // cookie jar has no numeric uid. The collector asks the bridge for
-    // 'toutiao.account' (local account_db) and proceeds with that uid.
-    const { fakeFetch } = makeFakeFetch([
-      [
-        "passport/account/info/v2",
-        { body: JSON.stringify({ message: "error", data: { error_code: 16, description: "该应用无权限" } }) },
-      ],
-      ...HAPPY_RESPONSES.slice(1),
-    ]);
-    const sign = {
-      warmUp: vi.fn(async () => {}),
-      signUrl: vi.fn(async (url) => {
-        const u = new URL(String(url));
-        u.searchParams.set("_signature", "BRIDGE_SIG");
-        return u;
-      }),
-      shutdown: vi.fn(async () => {}),
-    };
-    const bridge = {
-      invoke: vi.fn(async (m) => {
-        if (m === "toutiao.cookies") return { cookie: "sessionid=abc", uid: null, diagnostic: {} };
-        if (m === "toutiao.account") return { uid: "92585279158", nickname: "小明", secUid: "MS4w" };
-        throw new Error("unknown " + m);
-      }),
-    };
-    const client = new ToutiaoApiClient({ fetch: fakeFetch, signProvider: sign });
-    const r = await collect(bridge, { apiClient: client, signProvider: sign, stagingDir: os.tmpdir() });
-    expect(r.profileFetchFailed).toBe(true);
-    expect(r.profileSource).toBe("local-account-db");
-    expect(r.uid).toBe("92585279158");
-    expect(r.nickname).toBe("小明");
-    expect(r.lastErrorCode).toBe(16); // headline web error preserved
-    expect(r.eventCounts.profile).toBe(1); // profile event from local account
-    expect(r.eventCounts.feed).toBe(1);
-    expect(r.eventCounts.collection).toBe(1);
-    expect(r.eventCounts.search).toBe(1);
-  });
-});
-
-describe("collect — bridge warmUp failure", () => {
-  it("tolerates warmUp throw (falls through to fallback path)", async () => {
-    const { fakeFetch } = makeFakeFetch(HAPPY_RESPONSES);
-    const sign = {
-      warmUp: vi.fn(async () => {
-        throw new Error("toutiao.com 403 — anti-bot blocked");
-      }),
-      signUrl: vi.fn(async () => null),
-      shutdown: vi.fn(async () => {}),
-    };
-    const client = new ToutiaoApiClient({
-      fetch: fakeFetch,
-      signProvider: sign,
-    });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      signProvider: sign,
-      stagingDir: os.tmpdir(),
-    });
-    expect(r.profileFetchFailed).toBe(false); // profile uses no _sig
-    expect(r.eventCounts.feed).toBe(0); // signed endpoints fall through
-    expect(client._fallbackHits).toBe(3);
-    expect(sign.shutdown).toHaveBeenCalledOnce();
-  });
-});
-
-describe("collect — malformed bridge payload", () => {
-  it("throws when bridge.invoke returns no cookie", async () => {
-    const bridge = { invoke: vi.fn(async () => ({ uid: "1" })) };
-    await expect(
-      collect(bridge, { stagingDir: os.tmpdir() }),
-    ).rejects.toThrow(/malformed payload/);
-  });
-
-  it("throws when bridge missing invoke", async () => {
-    await expect(collect({}, {})).rejects.toThrow(
-      /bridge must expose invoke/,
-    );
-  });
-});
-
-describe("collect — signProviderUsed diagnostic", () => {
-  it("reports class name when bridge present", async () => {
-    const { fakeFetch } = makeFakeFetch(HAPPY_RESPONSES);
-    class ToutiaoSignBridge {
-      constructor() {
-        this.warmUp = vi.fn(async () => {});
-        this.signUrl = vi.fn(async (url) => {
-          const u = new URL(String(url));
-          u.searchParams.set("_signature", "X");
-          return u;
-        });
-        this.shutdown = vi.fn(async () => {});
-      }
-    }
-    const sign = new ToutiaoSignBridge();
-    const client = new ToutiaoApiClient({
-      fetch: fakeFetch,
-      signProvider: sign,
-    });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      signProvider: sign,
-      stagingDir: os.tmpdir(),
-    });
-    expect(r.signProviderUsed).toBe("ToutiaoSignBridge");
-  });
-
-  it("reports 'none' when no bridge", async () => {
-    const { fakeFetch } = makeFakeFetch(HAPPY_RESPONSES);
-    const client = new ToutiaoApiClient({ fetch: fakeFetch });
-    const r = await collect(makeBridge(COOKIE_PAYLOAD), {
-      apiClient: client,
-      stagingDir: os.tmpdir(),
-    });
-    expect(r.signProviderUsed).toBe("none");
-  });
-});
-
-describe("collectAndSync", () => {
-  it("orchestrates collect + registry.syncAdapter + cleanup", async () => {
-    const { fakeFetch } = makeFakeFetch(HAPPY_RESPONSES);
-    const client = new ToutiaoApiClient({ fetch: fakeFetch });
-    const registry = {
-      syncAdapter: vi.fn(async (name) => ({ adapter: name, status: "ok" })),
-    };
-    const r = await collectAndSync(makeBridge(COOKIE_PAYLOAD), registry, {
-      apiClient: client,
-      stagingDir: os.tmpdir(),
-    });
-    expect(registry.syncAdapter).toHaveBeenCalledWith(
-      "social-toutiao",
-      expect.objectContaining({ inputPath: expect.stringContaining(".json") }),
-    );
-    expect(r.adapter).toBe("social-toutiao");
-    expect(r.toutiao.uid).toBe("12345");
-    expect(r.toutiao.eventCounts.profile).toBe(1);
-  });
-});