@chainlesschain/personal-data-hub 0.4.29 → 0.4.30

package/__tests__/adapters/social-csdn.test.js

@@ -1,175 +0,0 @@
-"use strict";
-
-import { describe, it, expect } from "vitest";
-const fs = require("node:fs");
-const path = require("node:path");
-const os = require("node:os");
-const crypto = require("node:crypto");
-
-const {
-  CsdnAdapter,
-  extractList,
-  NAME,
-  VERSION,
-  SNAPSHOT_SCHEMA_VERSION,
-} = require("../../lib/adapters/social-csdn");
-
-function writeTmp(content) {
-  const p = path.join(os.tmpdir(), `cc-csdn-${crypto.randomUUID()}.json`);
-  fs.writeFileSync(p, content, "utf-8");
-  return p;
-}
-async function collect(gen) {
-  const out = [];
-  for await (const x of gen) out.push(x);
-  return out;
-}
-
-const COOKIES = "UserName=alice; UserToken=xyz";
-
-const SNAP = JSON.stringify({
-  schemaVersion: 1,
-  snapshottedAt: 1716383000000,
-  account: { username: "alice", name: "Alice" },
-  events: [
-    { kind: "article", id: "article-101", articleId: "101", title: "<p>Vue 源码解析</p>", viewCount: 999, collectCount: 50, createdTime: 1716300000, url: "https://blog.csdn.net/alice/article/details/101" },
-    { kind: "favourite", id: "fav-202", itemId: "202", title: "Rust 入门", url: "https://x/202", source: "blog", capturedAt: 1716310000000 },
-    { kind: "follow", id: "follow-bob", username: "bob", name: "Bob", capturedAt: 1716320000000 },
-  ],
-});
-
-describe("constants + extractList", () => {
-  it("name/version/schema", () => {
-    expect(NAME).toBe("social-csdn");
-    expect(VERSION).toBe("0.1.0");
-    expect(SNAPSHOT_SCHEMA_VERSION).toBe(1);
-  });
-  it("extractList tolerant", () => {
-    expect(extractList({ list: [{ id: 1 }] })).toHaveLength(1);
-    expect(extractList({ data: { list: [{ id: 1 }] } })).toHaveLength(1);
-    expect(extractList({ data: { records: [{ id: 1 }] } })).toHaveLength(1);
-    expect(extractList({})).toEqual([]);
-  });
-});
-
-describe("CsdnAdapter snapshot mode", () => {
-  it("authenticate validates inputPath", async () => {
-    const p = writeTmp(SNAP);
-    try {
-      const a = new CsdnAdapter();
-      expect((await a.authenticate({ inputPath: p })).mode).toBe("snapshot-file");
-      expect((await a.authenticate({ inputPath: path.join(os.tmpdir(), "no-csdn.json") })).reason).toBe("INPUT_PATH_UNREADABLE");
-    } finally {
-      fs.unlinkSync(p);
-    }
-  });
-
-  it("sync 3 kinds + normalize article→post(html-stripped)/favourite→like/follow→person", async () => {
-    const p = writeTmp(SNAP);
-    try {
-      const a = new CsdnAdapter();
-      const items = await collect(a.sync({ inputPath: p }));
-      expect(items.map((x) => x.kind)).toEqual(["article", "favourite", "follow"]);
-
-      const art = a.normalize(items[0]);
-      expect(art.events[0].subtype).toBe("post");
-      expect(art.events[0].content.text).toBe("Vue 源码解析"); // html stripped
-      expect(art.events[0].extra.viewCount).toBe(999);
-      expect(art.events[0].extra.collectCount).toBe(50);
-
-      const fav = a.normalize(items[1]);
-      expect(fav.events[0].subtype).toBe("like");
-      expect(fav.events[0].content.title).toBe("Rust 入门");
-
-      const fol = a.normalize(items[2]);
-      expect(fol.persons[0].subtype).toBe("contact");
-      expect(fol.persons[0].names).toEqual(["Bob"]);
-      expect(fol.persons[0].identifiers["csdn-username"]).toEqual(["bob"]);
-    } finally {
-      fs.unlinkSync(p);
-    }
-  });
-
-  it("include filter + limit + schema mismatch + unknown kind", async () => {
-    const p = writeTmp(SNAP);
-    try {
-      const a = new CsdnAdapter();
-      expect((await collect(a.sync({ inputPath: p, include: { article: false, follow: false } }))).map((x) => x.kind)).toEqual(["favourite"]);
-      expect(await collect(a.sync({ inputPath: p, limit: 2 }))).toHaveLength(2);
-      expect(() => a.normalize({ kind: "bogus", payload: {} })).toThrow(/unknown kind/);
-    } finally {
-      fs.unlinkSync(p);
-    }
-    const bad = writeTmp(JSON.stringify({ schemaVersion: 9, events: [] }));
-    try {
-      const a = new CsdnAdapter();
-      await expect(collect(a.sync({ inputPath: bad }))).rejects.toThrow(/schemaVersion mismatch/);
-    } finally {
-      fs.unlinkSync(bad);
-    }
-  });
-});
-
-describe("CsdnAdapter cookie-api mode", () => {
-  it("authenticate requires username", async () => {
-    const noU = new CsdnAdapter({ account: { cookies: COOKIES } });
-    expect((await noU.authenticate()).reason).toBe("NO_ACCOUNT_USERNAME");
-    const ok = new CsdnAdapter({ account: { cookies: COOKIES, username: "alice" } });
-    expect(await ok.authenticate()).toEqual({ ok: true, account: "alice", mode: "cookie" });
-  });
-
-  it("sync fetches articles/favourites/followees, normalizes", async () => {
-    const byUrl = (u) => (u.includes("get-business-list") ? "articles" : u.includes("favorite") ? "favourites" : "followees");
-    const data = {
-      articles: [{ articleId: "A1", title: "Go 并发", viewCount: 10, createdTime: 1716300000 }],
-      favourites: [{ id: "F1", title: "K8s 实践", source: "blog", created_at: 1716310000 }],
-      followees: [{ username: "carol", name: "Carol" }],
-    };
-    const calls = [];
-    const a = new CsdnAdapter({
-      account: { cookies: COOKIES, username: "alice" },
-      fetchFn: async ({ url, cookies, query, sign }) => {
-        const k = byUrl(url);
-        calls.push({ k, cookies, page: query.page, sign });
-        return { data: { list: query.page === 1 ? data[k] : [] } };
-      },
-    });
-    const items = await collect(a.sync({}));
-    expect(items.map((x) => x.kind).sort()).toEqual(["article", "favourite", "follow"]);
-    expect(calls.every((c) => c.cookies === COOKIES && c.sign === null)).toBe(true);
-    const art = a.normalize(items.find((x) => x.kind === "article"));
-    expect(art.events[0].content.title).toBe("Go 并发");
-    const fol = a.normalize(items.find((x) => x.kind === "follow"));
-    expect(fol.persons[0].names).toEqual(["Carol"]);
-  });
-
-  it("invokes signProvider", async () => {
-    const signCalls = [];
-    const a = new CsdnAdapter({
-      account: { cookies: COOKIES, username: "alice" },
-      fetchFn: async ({ query }) => ({ list: query.page === 1 ? [{ articleId: "A1", title: "x" }] : [] }),
-      signProvider: async (ctx) => { signCalls.push(ctx); return "sig"; },
-    });
-    const items = await collect(a.sync({ include: { favourite: false, follow: false } }));
-    expect(items.length).toBeGreaterThan(0);
-    expect(signCalls.length).toBeGreaterThan(0);
-    expect(signCalls[0].cookies).toBe(COOKIES);
-  });
-
-  it("limit + empty/login + default fetch + no input", async () => {
-    const a1 = new CsdnAdapter({
-      account: { cookies: COOKIES, username: "alice" },
-      fetchFn: async ({ query }) => ({ list: query.page === 1 ? [{ articleId: "A1", title: "a" }, { articleId: "A2", title: "b" }] : [] }),
-    });
-    expect(await collect(a1.sync({ limit: 1 }))).toHaveLength(1);
-
-    const a2 = new CsdnAdapter({ account: { cookies: COOKIES, username: "alice" }, fetchFn: async () => "<html>login</html>" });
-    expect(await collect(a2.sync({}))).toEqual([]);
-
-    const a3 = new CsdnAdapter({ account: { cookies: COOKIES, username: "alice" } });
-    await expect(collect(a3.sync({}))).rejects.toThrow(/no fetchFn configured/);
-
-    const a4 = new CsdnAdapter();
-    await expect(collect(a4.sync({}))).rejects.toThrow(/needs opts.inputPath/);
-  });
-});

package/__tests__/adapters/social-dongchedi.test.js

@@ -1,165 +0,0 @@
-"use strict";
-
-import { describe, it, expect } from "vitest";
-const fs = require("node:fs");
-const path = require("node:path");
-const os = require("node:os");
-const crypto = require("node:crypto");
-
-const {
-  DongchediAdapter,
-  extractData,
-  isEnd,
-  NAME,
-  VERSION,
-  SNAPSHOT_SCHEMA_VERSION,
-} = require("../../lib/adapters/social-dongchedi");
-
-function writeTmp(content) {
-  const p = path.join(os.tmpdir(), `cc-dcd-${crypto.randomUUID()}.json`);
-  fs.writeFileSync(p, content, "utf-8");
-  return p;
-}
-async function collect(gen) {
-  const out = [];
-  for await (const x of gen) out.push(x);
-  return out;
-}
-
-const COOKIES = "tt_webid=abc; sessionid=xyz";
-
-const SNAP = JSON.stringify({
-  schemaVersion: 1,
-  snapshottedAt: 1716383000000,
-  account: { userId: "u1" },
-  events: [
-    { kind: "favourite", id: "fav-1", itemId: "G1", title: "2026 新能源车横评", contentType: "article", url: "https://x/G1", capturedAt: 1716300000000 },
-    { kind: "follow", id: "follow-S1", followId: "S1", name: "理想 L 系列", followType: "series", capturedAt: 1716320000000 },
-  ],
-});
-
-describe("constants + helpers", () => {
-  it("name/version/schema", () => {
-    expect(NAME).toBe("social-dongchedi");
-    expect(VERSION).toBe("0.1.0");
-    expect(SNAPSHOT_SCHEMA_VERSION).toBe(1);
-  });
-  it("extractData tolerant", () => {
-    expect(extractData({ data: [{ id: 1 }] })).toHaveLength(1);
-    expect(extractData({ data: { favorite_list: [{ id: 1 }] } })).toHaveLength(1);
-    expect(extractData({ data: { follow_list: [{ id: 1 }] } })).toHaveLength(1);
-    expect(extractData({})).toEqual([]);
-  });
-  it("isEnd reads has_more", () => {
-    expect(isEnd({ data: { has_more: false } })).toBe(true);
-    expect(isEnd({ has_more: 0 })).toBe(true);
-    expect(isEnd({ data: { has_more: true } })).toBe(false);
-  });
-});
-
-describe("DongchediAdapter snapshot mode", () => {
-  it("authenticate validates inputPath", async () => {
-    const p = writeTmp(SNAP);
-    try {
-      const a = new DongchediAdapter();
-      expect((await a.authenticate({ inputPath: p })).mode).toBe("snapshot-file");
-      expect((await a.authenticate({ inputPath: path.join(os.tmpdir(), "no-dcd.json") })).reason).toBe("INPUT_PATH_UNREADABLE");
-    } finally {
-      fs.unlinkSync(p);
-    }
-  });
-
-  it("sync 2 kinds + normalize favourite→like / follow→person", async () => {
-    const p = writeTmp(SNAP);
-    try {
-      const a = new DongchediAdapter();
-      const items = await collect(a.sync({ inputPath: p }));
-      expect(items.map((x) => x.kind)).toEqual(["favourite", "follow"]);
-
-      const fav = a.normalize(items[0]);
-      expect(fav.events[0].subtype).toBe("like");
-      expect(fav.events[0].content.title).toBe("收藏: 2026 新能源车横评");
-      expect(fav.events[0].extra.contentType).toBe("article");
-
-      const fol = a.normalize(items[1]);
-      expect(fol.persons[0].subtype).toBe("contact");
-      expect(fol.persons[0].names).toEqual(["理想 L 系列"]);
-      expect(fol.persons[0].identifiers["dongchedi-id"]).toEqual(["S1"]);
-      expect(fol.persons[0].extra.followType).toBe("series");
-    } finally {
-      fs.unlinkSync(p);
-    }
-  });
-
-  it("include + limit + schema mismatch + unknown kind", async () => {
-    const p = writeTmp(SNAP);
-    try {
-      const a = new DongchediAdapter();
-      expect((await collect(a.sync({ inputPath: p, include: { favourite: false } }))).map((x) => x.kind)).toEqual(["follow"]);
-      expect(await collect(a.sync({ inputPath: p, limit: 1 }))).toHaveLength(1);
-      expect(() => a.normalize({ kind: "bogus", payload: {} })).toThrow(/unknown kind/);
-    } finally {
-      fs.unlinkSync(p);
-    }
-    const bad = writeTmp(JSON.stringify({ schemaVersion: 9, events: [] }));
-    try {
-      const a = new DongchediAdapter();
-      await expect(collect(a.sync({ inputPath: bad }))).rejects.toThrow(/schemaVersion mismatch/);
-    } finally {
-      fs.unlinkSync(bad);
-    }
-  });
-});
-
-describe("DongchediAdapter cookie-api mode", () => {
-  it("authenticate cookie (userId optional)", async () => {
-    const a = new DongchediAdapter({ account: { cookies: COOKIES } });
-    expect(await a.authenticate()).toEqual({ ok: true, account: null, mode: "cookie" });
-  });
-
-  it("sync fetches favourites + follows, normalizes", async () => {
-    const byUrl = (u) => (u.includes("favorite") ? "favourite" : "follow");
-    const data = {
-      favourite: [{ group_id: "G1", title: "试驾视频", content_type: "video", create_time: 1716300000 }],
-      follow: [{ series_id: "S9", series_name: "比亚迪汉", follow_time: 1716320000 }],
-    };
-    const calls = [];
-    const a = new DongchediAdapter({
-      account: { cookies: COOKIES, userId: "u1" },
-      fetchFn: async ({ url, cookies, query, sign }) => {
-        const k = byUrl(url);
-        calls.push({ k, cookies, offset: query.offset, sign });
-        return { data: { list: query.offset === 0 ? data[k] : [], has_more: false } };
-      },
-    });
-    const items = await collect(a.sync({}));
-    expect(items.map((x) => x.kind).sort()).toEqual(["favourite", "follow"]);
-    expect(calls.every((c) => c.cookies === COOKIES && c.sign === null)).toBe(true);
-    const fav = a.normalize(items.find((x) => x.kind === "favourite"));
-    expect(fav.events[0].content.title).toBe("收藏: 试驾视频");
-    const fol = a.normalize(items.find((x) => x.kind === "follow"));
-    expect(fol.persons[0].names).toEqual(["比亚迪汉"]);
-    expect(fol.persons[0].extra.followType).toBe("series");
-  });
-
-  it("invokes signProvider + limit + empty + default fetch + no input", async () => {
-    const signCalls = [];
-    const a = new DongchediAdapter({
-      account: { cookies: COOKIES },
-      fetchFn: async ({ query }) => ({ data: { list: query.offset === 0 ? [{ group_id: "G1", title: "x" }, { group_id: "G2", title: "y" }] : [], has_more: false } }),
-      signProvider: async (ctx) => { signCalls.push(ctx); return "x-bogus"; },
-    });
-    expect(await collect(a.sync({ limit: 1, include: { follow: false } }))).toHaveLength(1);
-    expect(signCalls.length).toBeGreaterThan(0);
-    expect(signCalls[0].cookies).toBe(COOKIES);
-
-    const a2 = new DongchediAdapter({ account: { cookies: COOKIES }, fetchFn: async () => "<html>login</html>" });
-    expect(await collect(a2.sync({}))).toEqual([]);
-
-    const a3 = new DongchediAdapter({ account: { cookies: COOKIES } });
-    await expect(collect(a3.sync({}))).rejects.toThrow(/no fetchFn configured/);
-
-    const a4 = new DongchediAdapter();
-    await expect(collect(a4.sync({}))).rejects.toThrow(/needs opts.inputPath/);
-  });
-});

package/__tests__/adapters/social-douyin-adb-aweme-detail.test.js

@@ -1,165 +0,0 @@
-/**
- * Douyin aweme-detail title resolver tests (real-device-driven 2026-06-11:
- * web detail endpoint returns aweme_detail{desc,author,duration} with no signing).
- * fetch injected — no network.
- */
-"use strict";
-
-import { describe, it, expect, vi } from "vitest";
-
-const {
-  AwemeDetailClient,
-} = require("../../lib/adapters/social-douyin-adb/aweme-detail-client");
-const {
-  collectWatchHistory,
-} = require("../../lib/adapters/social-douyin-adb/collector");
-const { DouyinAdapter } = require("../../lib/adapters/social-douyin");
-const { partitionBatch } = require("../../lib/batch");
-
-function makeFetch(byAid) {
-  return async (url) => {
-    const m = /aweme_id=(\d+)/.exec(url);
-    const aid = m && m[1];
-    const payload = byAid[aid];
-    if (payload === undefined) return { ok: false, status: 404, text: async () => "nf" };
-    return {
-      ok: payload.status ? payload.status >= 200 && payload.status < 300 : true,
-      status: payload.status || 200,
-      text: async () => (typeof payload.body === "string" ? payload.body : JSON.stringify(payload.body)),
-    };
-  };
-}
-const noSleep = () => Promise.resolve();
-
-describe("AwemeDetailClient.fetchDetail", () => {
-  it("parses aweme_detail → {desc, author, durationMs, createTime}; sends aweme_id + webapp params", async () => {
-    let seenUrl;
-    const fetch = async (url) => {
-      seenUrl = url;
-      return { ok: true, status: 200, text: async () => JSON.stringify({
-        status_code: 0,
-        aweme_detail: { desc: "洋气婆婆和她的土狗儿媳", author: { nickname: "任集" }, duration: 9200, create_time: 1780112750 },
-      }) };
-    };
-    const c = new AwemeDetailClient({ fetch, sleep: noSleep });
-    const d = await c.fetchDetail("7645526043227334246");
-    expect(d).toEqual({ awemeId: "7645526043227334246", desc: "洋气婆婆和她的土狗儿媳", author: "任集", durationMs: 9200, createTime: 1780112750 });
-    expect(seenUrl).toContain("aweme_id=7645526043227334246");
-    expect(seenUrl).toContain("device_platform=webapp");
-    expect(seenUrl).toContain("aid=6383");
-  });
-
-  it("status_code != 0 → null + lastError", async () => {
-    const fetch = makeFetch({ 1: { body: { status_code: 8, status_msg: "risk" } } });
-    const c = new AwemeDetailClient({ fetch, sleep: noSleep });
-    expect(await c.fetchDetail("1")).toBe(null);
-    expect(c.lastErrorCode).toBe(8);
-  });
-
-  it("missing aweme_detail (deleted/private) → null -5", async () => {
-    const fetch = makeFetch({ 1: { body: { status_code: 0 } } });
-    const c = new AwemeDetailClient({ fetch, sleep: noSleep });
-    expect(await c.fetchDetail("1")).toBe(null);
-    expect(c.lastErrorCode).toBe(-5);
-  });
-
-  it("HTTP non-2xx → null with status", async () => {
-    const fetch = makeFetch({ 1: { status: 444, body: "blocked" } });
-    const c = new AwemeDetailClient({ fetch, sleep: noSleep });
-    expect(await c.fetchDetail("1")).toBe(null);
-    expect(c.lastErrorCode).toBe(444);
-  });
-});
-
-describe("AwemeDetailClient.resolveMany", () => {
-  it("dedups ids, caps at limit, skips per-id failures", async () => {
-    const calls = [];
-    const fetch = async (url) => {
-      const aid = /aweme_id=(\d+)/.exec(url)[1];
-      calls.push(aid);
-      const ok = { 11: true, 22: true }[aid];
-      return { ok: true, status: 200, text: async () => JSON.stringify(
-        ok ? { status_code: 0, aweme_detail: { desc: "d" + aid, author: { nickname: "a" } } }
-           : { status_code: 0 }, // 33 → no aweme_detail → skipped
-      ) };
-    };
-    const c = new AwemeDetailClient({ fetch, sleep: noSleep });
-    const map = await c.resolveMany(["11", "11", "22", "33"], { limit: 10 });
-    expect(calls).toEqual(["11", "22", "33"]); // deduped (one 11)
-    expect([...map.keys()].sort()).toEqual(["11", "22"]); // 33 skipped
-    expect(map.get("11").desc).toBe("d11");
-  });
-
-  it("respects limit (stops early)", async () => {
-    const calls = [];
-    const fetch = async (url) => {
-      calls.push(/aweme_id=(\d+)/.exec(url)[1]);
-      return { ok: true, status: 200, text: async () => JSON.stringify({ status_code: 0, aweme_detail: { desc: "x" } }) };
-    };
-    const c = new AwemeDetailClient({ fetch, sleep: noSleep });
-    await c.resolveMany(["1", "2", "3", "4", "5"], { limit: 2 });
-    expect(calls).toEqual(["1", "2"]);
-  });
-});
-
-describe("collectWatchHistory --resolve-titles integration", () => {
-  it("attaches title/author/duration → normalizeHistory shows real content", async () => {
-    const fs = require("node:fs");
-    const os = require("node:os");
-    const bridge = {
-      invoke: vi.fn(async (m) =>
-        m === "douyin.watch-history"
-          ? {
-              uid: "92585448288",
-              records: [
-                { awemeId: "7645526043227334246", capturedAt: 1780112750000, enterFrom: "homepage_hot" },
-              ],
-            }
-          : (() => { throw new Error("unknown " + m); })(),
-      ),
-    };
-    const detailClient = new AwemeDetailClient({
-      sleep: () => Promise.resolve(),
-      fetch: makeFetch({
-        "7645526043227334246": { body: { status_code: 0, aweme_detail: { desc: "洋气婆婆和她的土狗儿媳", author: { nickname: "任集" }, duration: 9200 } } },
-      }),
-    });
-    const r = await collectWatchHistory(bridge, {
-      stagingDir: os.tmpdir(),
-      now: () => 1781000000000,
-      resolveTitles: true,
-      _detailClient: detailClient,
-    });
-    expect(r.titlesResolved).toBe(1);
-    const snap = JSON.parse(fs.readFileSync(r.snapshotPath, "utf-8"));
-    try {
-      expect(snap.events[0].title).toBe("洋气婆婆和她的土狗儿媳");
-      const a = new DouyinAdapter();
-      const batch = a.normalize({
-        adapter: "social-douyin",
-        kind: "history",
-        originalId: "douyin:history:1",
-        capturedAt: 1780112750000,
-        payload: { ...snap.events[0], account: snap.account },
-      });
-      expect(partitionBatch(batch).invalidReasons).toHaveLength(0);
-      expect(batch.events[0].content.title).toBe("洋气婆婆和她的土狗儿媳");
-      expect(batch.events[0].extra.author).toBe("任集");
-      expect(batch.events[0].extra.duration).toBe(9200);
-      expect(batch.events[0].extra.enterFrom).toBe("homepage_hot");
-    } finally {
-      fs.unlinkSync(r.snapshotPath);
-    }
-  });
-
-  it("without resolveTitles, no network + title stays unresolved", async () => {
-    const os = require("node:os");
-    const fs = require("node:fs");
-    const bridge = { invoke: vi.fn(async () => ({ uid: "1", records: [{ awemeId: "999", capturedAt: 1, enterFrom: "x" }] })) };
-    const r = await collectWatchHistory(bridge, { stagingDir: os.tmpdir(), now: () => 1, resolveTitles: false });
-    expect(r.titlesResolved).toBe(0);
-    const snap = JSON.parse(fs.readFileSync(r.snapshotPath, "utf-8"));
-    fs.unlinkSync(r.snapshotPath);
-    expect(snap.events[0].title).toBeUndefined();
-  });
-});