@chainlesschain/personal-data-hub 0.4.28 → 0.4.29

package/README.md

@@ -16,13 +16,21 @@ middleware.
 > Ollama / Volcengine / Anthropic / Gemini / DeepSeek), **CcKgSink**, **CcRagSink**
 > — injected at the desktop/CLI entry so this package stays decoupled (Phase 3.5).
 >
-> **51 adapters are now live** (no longer "later phases"): Email IMAP,
+> **89 adapters are now live** (no longer "later phases"): Email IMAP,
 > Alipay bill, 9 AI-chat vendors, WeChat / QQ / Weibo / Bilibili / Douyin /
-> Xiaohongshu / Toutiao / Kuaishou social, Telegram / WhatsApp messaging,
-> Taobao / JD / Meituan / Pinduoduo shopping, Amap / Baidu-map / Tencent-map /
-> Ctrip / 12306 travel, system-data (contacts / calls / sms / location),
+> Xiaohongshu / Toutiao / Kuaishou / Douban social, Telegram / WhatsApp messaging,
+> Taobao / JD / Meituan / Pinduoduo / Eleme / Xianyu / Vipshop shopping, Amap /
+> Baidu-map / Tencent-map / Ctrip / 12306 / Didi travel, Kugou / Ximalaya audio,
+> Keep / Joyrun fitness, system-data (contacts / calls / sms / location),
 > and the developer-activity set (git / shell / vscode / browser-history /
-> local-files / win-recent).
+> local-files / win-recent). See `lib/adapters/` for the full list.
+>
+> **On-device root forensics (rooted devices):** beyond cookie/sign-based
+> collection, PDH can pull a logged-in app's local encrypted DB directly via
+> **method B** (key-free `/proc/<pid>/mem` memory scan — engine-agnostic,
+> anti-debug-resistant) or **method C** (frida `sqlcipher_export` online decrypt),
+> plus a SQLite leaf-page **salvager** (`--unaligned`) that recovers plaintext
+> pages from corrupt mem dumps. See `docs/internal/pdh-db-decryption-runbook.md`.
 >
 > **New in v0.4.0 (v5.0.3.99):** adapter **readiness** — split out from the
 > loose `healthCheck` sync gate into a real ready/needs_setup/unavailable

package/__tests__/adapters/social-douyin-adb-usage-profile.test.js

@@ -0,0 +1,229 @@
+/**
+ * Douyin usage-profile reader tests (real-device-driven 2026-06-18: the user's
+ * exported 1128_feature_engineering.db has FEInternalUserActivityTable = 81
+ * rows ≈ 24 days, 175 opens, ~108h, peak 12-17h).
+ *
+ * Two layers:
+ *  - pure aggregation via an injected fake Database (no native driver needed);
+ *  - a real better-sqlite3 db + real LocalVault round-trip proving the
+ *    hand-built event passes schema validation, is searchable, and re-ingest
+ *    dedups on the stable originalId.
+ */
+"use strict";
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const { LocalVault } = require("../../lib/vault");
+const { generateKeyHex } = require("../../lib/key-providers");
+const {
+  USAGE_TABLE,
+  readDouyinUsageProfile,
+  summarizeUsageProfile,
+  buildUsageProfileEvents,
+  usageProfileToVault,
+  _internals,
+} = require("../../lib/adapters/social-douyin-adb/usage-profile-reader");
+
+// ── pure aggregation with an injected fake Database ───────────────────
+function makeFakeDb(rows, { table = USAGE_TABLE } = {}) {
+  const cols = [
+    "id",
+    "timestamp",
+    "open_app_count",
+    "total_duration",
+    ...Array.from({ length: 24 }, (_v, h) => `launch_hour_${h}`),
+  ];
+  return class FakeDb {
+    constructor() {}
+    prepare(sql) {
+      return {
+        get: (arg) => {
+          if (/sqlite_master/.test(sql)) {
+            return arg === table ? { name: table } : undefined;
+          }
+          return undefined;
+        },
+        all: () => {
+          if (/table_info/.test(sql)) return cols.map((name) => ({ name }));
+          if (/FROM "/.test(sql)) return rows;
+          return [];
+        },
+      };
+    }
+    close() {}
+  };
+}
+
+function row({ ts, opens = 1, durMs = 0, hours = {} }) {
+  const r = { id: 1, timestamp: ts, open_app_count: opens, total_duration: durMs };
+  for (let h = 0; h < 24; h++) r[`launch_hour_${h}`] = hours[h] || 0;
+  return r;
+}
+
+const DAY = 86_400_000;
+
+describe("readDouyinUsageProfile (injected fake db)", () => {
+  it("aggregates opens, duration, hour histogram, peak hour + bucket, distinct days", () => {
+    const base = Math.floor(1781000000000 / 1000); // seconds epoch
+    const Db = makeFakeDb([
+      row({ ts: base, opens: 2, durMs: 3_600_000, hours: { 13: 3, 9: 1 } }),
+      row({ ts: base + 86_400, opens: 1, durMs: 1_800_000, hours: { 14: 2, 20: 1 } }),
+    ]);
+    const p = readDouyinUsageProfile("x.db", { _databaseClass: Db });
+    expect(p.sessions).toBe(2);
+    expect(p.days).toBe(2);
+    expect(p.totalOpens).toBe(3);
+    expect(p.totalDurationMs).toBe(5_400_000);
+    expect(p.hourHistogram[13]).toBe(3);
+    expect(p.hourHistogram[14]).toBe(2);
+    expect(p.peakHour).toBe(13); // 3 launches is the single max hour
+    expect(p.peakBucket).toBe("12-17h"); // 13+14 = 5 dominates
+    expect(p.bucketTotals["12-17h"]).toBe(5);
+    expect(p.bucketTotals["18-23h"]).toBe(1);
+    expect(p.from).toBe(base * 1000);
+    expect(p.to).toBe((base + 86_400) * 1000);
+  });
+
+  it("returns an empty profile when the table is absent", () => {
+    const Db = makeFakeDb([], { table: "SomeOtherTable" });
+    const p = readDouyinUsageProfile("x.db", { _databaseClass: Db });
+    expect(p.sessions).toBe(0);
+    expect(p.peakBucket).toBe(null);
+    expect(p.hourHistogram).toHaveLength(24);
+    expect(p.totalDurationMs).toBe(0);
+  });
+
+  it("counts the same calendar day once even across multiple sessions", () => {
+    const base = Math.floor(1781000000000 / 1000);
+    const Db = makeFakeDb([
+      row({ ts: base, hours: { 10: 1 } }),
+      row({ ts: base + 3600, hours: { 11: 1 } }), // same UTC day
+    ]);
+    const p = readDouyinUsageProfile("x.db", { _databaseClass: Db });
+    expect(p.days).toBe(1);
+    expect(p.sessions).toBe(2);
+  });
+
+  it("toEpochMs treats >1e12 as ms else seconds; rejects junk", () => {
+    expect(_internals.toEpochMs(1781000000)).toBe(1781000000000);
+    expect(_internals.toEpochMs(1781000000000)).toBe(1781000000000);
+    expect(_internals.toEpochMs(0)).toBe(null);
+    expect(_internals.toEpochMs("nope")).toBe(null);
+  });
+});
+
+describe("summarizeUsageProfile + buildUsageProfileEvents", () => {
+  it("summary is empty-safe and renders hours + peak", () => {
+    expect(summarizeUsageProfile(null)).toMatch(/无数据/);
+    expect(summarizeUsageProfile({ sessions: 0 })).toMatch(/无数据/);
+    const txt = summarizeUsageProfile({
+      sessions: 81,
+      days: 24,
+      totalOpens: 175,
+      totalDurationMs: 388_440_000, // 107.9h
+      peakBucket: "12-17h",
+    });
+    expect(txt).toMatch(/24 天/);
+    expect(txt).toMatch(/175 次启动/);
+    expect(txt).toMatch(/107\.9 小时/);
+    expect(txt).toMatch(/12-17h/);
+  });
+
+  it("builds no events for an empty profile", () => {
+    expect(buildUsageProfileEvents({ sessions: 0 }).events).toHaveLength(0);
+    expect(buildUsageProfileEvents(null).events).toHaveLength(0);
+  });
+
+  it("builds one app-usage-profile event with stable originalId + histogram in extra", () => {
+    const profile = {
+      sessions: 81, days: 24, from: 1, to: 1781800000000,
+      totalOpens: 175, totalDurationMs: 388_440_000,
+      hourHistogram: new Array(24).fill(0), peakHour: 13, peakBucket: "12-17h",
+      bucketTotals: { "0-5h": 1, "6-11h": 81, "12-17h": 107, "18-23h": 75 },
+    };
+    const { events } = buildUsageProfileEvents(profile, { now: 1781900000000 });
+    expect(events).toHaveLength(1);
+    const e = events[0];
+    expect(e.subtype).toBe("other");
+    expect(e.source.adapter).toBe("social-douyin");
+    expect(e.source.originalId).toBe("social-douyin:usage-profile");
+    expect(e.source.capturedBy).toBe("sqlite");
+    expect(e.occurredAt).toBe(1781800000000); // profile.to
+    expect(e.extra.kind).toBe("app-usage-profile");
+    expect(e.extra.peakBucket).toBe("12-17h");
+    expect(e.extra.bucketTotals["12-17h"]).toBe(107);
+    expect(Array.isArray(e.extra.hourHistogram)).toBe(true);
+  });
+});
+
+// ── real db + real vault round-trip (schema validation + dedup) ───────
+describe("usageProfileToVault — real sqlite + real vault", () => {
+  let dir, dbPath, vdir, vault;
+
+  beforeAll(() => {
+    const Database = require("better-sqlite3-multiple-ciphers");
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "douyin-usage-"));
+    dbPath = path.join(dir, "1128_feature_engineering.db");
+    const db = new Database(dbPath);
+    const hourCols = Array.from({ length: 24 }, (_v, h) => `launch_hour_${h} INTEGER`).join(", ");
+    db.exec(
+      `CREATE TABLE "${USAGE_TABLE}" (id INTEGER, timestamp INTEGER, ` +
+        `start_timestamp_ms INTEGER, end_timestamp_ms INTEGER, ` +
+        `open_app_count INTEGER, ${hourCols}, total_duration INTEGER)`,
+    );
+    const hzero = Array.from({ length: 24 }, () => 0);
+    const insCols = ["id", "timestamp", "start_timestamp_ms", "end_timestamp_ms", "open_app_count",
+      ...Array.from({ length: 24 }, (_v, h) => `launch_hour_${h}`), "total_duration"];
+    const ph = insCols.map(() => "?").join(",");
+    const ins = db.prepare(`INSERT INTO "${USAGE_TABLE}" (${insCols.join(",")}) VALUES (${ph})`);
+    const baseSec = Math.floor(1781000000000 / 1000);
+    // two sessions on two different days; 13h is the peak hour
+    const h1 = [...hzero]; h1[13] = 3; h1[9] = 1;
+    const h2 = [...hzero]; h2[14] = 2;
+    ins.run(1, baseSec, baseSec * 1000, baseSec * 1000 + 1000, 2, ...h1, 3_600_000);
+    ins.run(2, baseSec + 86_400, 0, 0, 1, ...h2, 1_800_000);
+    db.close();
+
+    vdir = fs.mkdtempSync(path.join(os.tmpdir(), "douyin-usage-vault-"));
+    vault = new LocalVault({ path: path.join(vdir, "v.db"), key: generateKeyHex() });
+    vault.open();
+  });
+
+  afterAll(() => {
+    try { vault.close(); } catch (_e) { /* best-effort */ }
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
+    try { fs.rmSync(vdir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
+  });
+
+  it("reads the real table, aggregates, and ingests one valid event", () => {
+    const profile = readDouyinUsageProfile(dbPath, {});
+    expect(profile.sessions).toBe(2);
+    expect(profile.days).toBe(2);
+    expect(profile.peakHour).toBe(13);
+    expect(profile.peakBucket).toBe("12-17h");
+
+    const r = usageProfileToVault(vault, dbPath, { now: 1781900000000 });
+    expect(r.ingested).toBe(1); // proves the hand-built event passed schema validation
+    expect(r.sessions).toBe(2);
+
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const mine = events.filter(
+      (e) => e.extra && e.extra.kind === "app-usage-profile",
+    );
+    expect(mine.length).toBe(1);
+    expect(mine[0].source.adapter).toBe("social-douyin");
+  });
+
+  it("re-ingest dedups on the stable originalId (no duplicate baseline)", () => {
+    usageProfileToVault(vault, dbPath, { now: 1781999999999 });
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const mine = events.filter(
+      (e) => e.extra && e.extra.kind === "app-usage-profile",
+    );
+    expect(mine.length).toBe(1); // still one — updated, not duplicated
+  });
+});

package/__tests__/adapters/social-douyin-adb-watch-history.test.js

@@ -7,10 +7,11 @@
  */
 "use strict";
 
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect, vi, beforeAll, afterAll } from "vitest";
 
 const {
   createDouyinWatchExtension,
+  watchHistoryToVault,
   VIDEO_RECORD_DB_REMOTE_PATH,
   _internals,
 } = require("../../lib/adapters/social-douyin-adb/watch-history-reader");
@@ -55,9 +56,9 @@ function makeFakeDb(tablesToRows) {
 }
 
 describe("readDouyinWatchHistory", () => {
-  it("picks the largest record_<uid> table (skips record_0), parses rows → ms", () => {
+  it("merges record_0 + record_<uid>, attributes uid to the largest uid table, parses rows → ms", () => {
     const Db = makeFakeDb({
-      record_0: [{ aid: "x", view_time_timestamp: 1, enter_from: "a" }],
+      record_0: [{ aid: "xrec0", view_time_timestamp: 1, enter_from: "a" }],
       record_92585448288: [
         { aid: "7480000000000000001", view_time_timestamp: 1717800000, enter_from: "homepage_hot" },
         { aid: "7480000000000000002", view_time_timestamp: 1717800600, enter_from: "homepage_follow" },
@@ -65,19 +66,40 @@ describe("readDouyinWatchHistory", () => {
     });
     const r = _internals.readDouyinWatchHistory("x.db", { _databaseClass: Db });
     expect(r.uid).toBe("92585448288");
-    expect(r.records).toHaveLength(2);
+    // record_0 is no longer dropped: 1 (record_0) + 2 (uid) = 3 merged records.
+    expect(r.records).toHaveLength(3);
+    // Most-recent first.
     expect(r.records[0]).toEqual({
-      awemeId: "7480000000000000001",
-      capturedAt: 1717800000 * 1000, // seconds → ms
-      enterFrom: "homepage_hot",
+      awemeId: "7480000000000000002",
+      capturedAt: 1717800600 * 1000,
+      enterFrom: "homepage_follow",
     });
+    const ids = r.records.map((x) => x.awemeId);
+    expect(ids).toContain("xrec0"); // the formerly-lost record_0 row
+    const rec0 = r.records.find((x) => x.awemeId === "xrec0");
+    expect(rec0).toEqual({ awemeId: "xrec0", capturedAt: 1000, enterFrom: "a" });
   });
 
-  it("returns {uid:null, records:[]} when only the anonymous record_0 exists", () => {
-    const Db = makeFakeDb({ record_0: [{ aid: "x", view_time_timestamp: 1 }] });
+  it("recovers history from record_0 alone (uid:null) — the bulk-in-record_0 device case", () => {
+    const Db = makeFakeDb({
+      record_0: [
+        { aid: "a1", view_time_timestamp: 1717800000, enter_from: "homepage_hot" },
+        { aid: "a2", view_time_timestamp: 1717800600, enter_from: "homepage_hot" },
+      ],
+    });
     const r = _internals.readDouyinWatchHistory("x.db", { _databaseClass: Db });
-    expect(r.uid).toBe(null);
-    expect(r.records).toEqual([]);
+    expect(r.uid).toBe(null); // no logged-in account table → no attribution
+    expect(r.records).toHaveLength(2); // but the watch history is still recovered
+    expect(r.records.map((x) => x.awemeId).sort()).toEqual(["a1", "a2"]);
+  });
+
+  it("dedups the same (aid, timestamp) appearing in two record_* tables", () => {
+    const Db = makeFakeDb({
+      record_0: [{ aid: "dup", view_time_timestamp: 1717800000, enter_from: "homepage_hot" }],
+      record_111: [{ aid: "dup", view_time_timestamp: 1717800000, enter_from: "homepage_hot" }],
+    });
+    const r = _internals.readDouyinWatchHistory("x.db", { _databaseClass: Db });
+    expect(r.records).toHaveLength(1);
   });
 
   it("toEpochMs treats >1e12 as ms, else seconds; rejects junk", () => {
@@ -190,3 +212,58 @@ describe("createDouyinWatchExtension contract", () => {
     await expect(createDouyinWatchExtension()({}, {})).rejects.toThrow(/ctx must provide/);
   });
 });
+
+// ── watchHistoryToVault: local-db → canonical BROWSE events → vault ────
+describe("watchHistoryToVault — real sqlite + real vault", () => {
+  const fs = require("node:fs");
+  const path = require("node:path");
+  const os = require("node:os");
+  const { LocalVault } = require("../../lib/vault");
+  const { generateKeyHex } = require("../../lib/key-providers");
+  let dir, dbPath, vdir, vault;
+
+  beforeAll(() => {
+    const Database = require("better-sqlite3-multiple-ciphers");
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "dy-watch-"));
+    dbPath = path.join(dir, "video_record.db");
+    const db = new Database(dbPath);
+    db.exec("CREATE TABLE record_0 (aid TEXT, view_time_timestamp INTEGER, enter_from TEXT)");
+    db.exec("CREATE TABLE record_92585448288 (aid TEXT, view_time_timestamp INTEGER, enter_from TEXT)");
+    db.prepare("INSERT INTO record_0 VALUES (?,?,?)").run("7644480728574545765", 1781706182375, "homepage_hot");
+    db.prepare("INSERT INTO record_92585448288 VALUES (?,?,?)").run("7480000000000000002", 1717800600000, "others_homepage");
+    db.close();
+
+    vdir = fs.mkdtempSync(path.join(os.tmpdir(), "dy-watch-vault-"));
+    vault = new LocalVault({ path: path.join(vdir, "v.db"), key: generateKeyHex() });
+    vault.open();
+  });
+
+  afterAll(() => {
+    try { vault.close(); } catch (_e) { /* best-effort */ }
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
+    try { fs.rmSync(vdir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
+  });
+
+  it("merges record_0 + uid table and ingests canonical BROWSE events", () => {
+    const r = watchHistoryToVault(vault, dbPath, { now: 1781900000000 });
+    expect(r.records).toBe(2); // record_0 row no longer dropped
+    expect(r.ingested).toBe(2);
+    expect(r.uid).toBe("92585448288");
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const browse = events.filter(
+      (e) => e.subtype === "browse" && e.source.adapter === "social-douyin",
+    );
+    expect(browse.length).toBe(2);
+    expect(browse.some((e) => e.extra.awemeId === "7644480728574545765")).toBe(true);
+    expect(browse.some((e) => e.extra.enterFrom === "homepage_hot")).toBe(true);
+  });
+
+  it("re-ingest dedups on the per-record originalId", () => {
+    watchHistoryToVault(vault, dbPath, { now: 1781999999999 });
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const browse = events.filter(
+      (e) => e.subtype === "browse" && e.source.adapter === "social-douyin",
+    );
+    expect(browse.length).toBe(2); // still two
+  });
+});

package/__tests__/adapters/social-toutiao-adb-article.test.js

@@ -0,0 +1,155 @@
+/**
+ * Toutiao article reader tests (real-device-driven 2026-06-18: the user's
+ * exported news_article.db `article` table = 48 feed-cache rows; title lives in
+ * the share_info JSON blob, not a column).
+ *
+ * Two layers: pure parsing via injected fake Database, + a real better-sqlite3
+ * db + real LocalVault round-trip proving the hand-built BROWSE events pass
+ * schema validation, are searchable, and re-ingest dedups on the stable
+ * originalId.
+ */
+"use strict";
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const { LocalVault } = require("../../lib/vault");
+const { generateKeyHex } = require("../../lib/key-providers");
+const {
+  ARTICLE_TABLE,
+  readToutiaoArticles,
+  buildArticleEvents,
+  articlesToVault,
+  _internals,
+} = require("../../lib/adapters/social-toutiao-adb/article-reader");
+
+function makeFakeDb(rows, { table = ARTICLE_TABLE } = {}) {
+  const cols = [
+    "group_id", "item_id", "share_info", "ext_json", "share_url",
+    "behot_time", "read_timestamp", "is_user_digg", "is_user_repin",
+  ];
+  return class FakeDb {
+    constructor() {}
+    prepare(sql) {
+      return {
+        get: (arg) => (/sqlite_master/.test(sql) ? (arg === table ? { name: table } : undefined) : undefined),
+        all: () => {
+          if (/table_info/.test(sql)) return cols.map((name) => ({ name }));
+          if (/FROM "/.test(sql)) return rows;
+          return [];
+        },
+      };
+    }
+    close() {}
+  };
+}
+
+describe("readToutiaoArticles (injected fake db)", () => {
+  it("parses title from share_info, strips the brand suffix, drops url tracking query", () => {
+    const Db = makeFakeDb([
+      {
+        group_id: 100, behot_time: 1781700000, read_timestamp: 0, is_user_digg: 1, is_user_repin: 0,
+        share_info: JSON.stringify({ title: "5月汽车出口延续快速增长态势 - 今日头条", share_url: "https://m.toutiao.com/g/100/?app=x&category_new=headline" }),
+        share_url: "https://m.toutiao.com/g/100/?category_new=headline",
+      },
+    ]);
+    const { articles } = readToutiaoArticles("x.db", { _databaseClass: Db });
+    expect(articles).toHaveLength(1);
+    expect(articles[0].title).toBe("5月汽车出口延续快速增长态势"); // suffix stripped
+    expect(articles[0].url).toBe("https://m.toutiao.com/g/100/"); // query dropped
+    expect(articles[0].category).toBe("headline");
+    expect(articles[0].digg).toBe(true);
+    expect(articles[0].behotTime).toBe(1781700000 * 1000);
+  });
+
+  it("falls back to ext_json.title when share_info has none, and skips untitled rows", () => {
+    const Db = makeFakeDb([
+      { group_id: 1, ext_json: JSON.stringify({ title: "来自 ext_json 的标题" }), share_info: "{}" },
+      { group_id: 2, share_info: "{}", ext_json: "{}" }, // untitled → dropped
+    ]);
+    const { articles } = readToutiaoArticles("x.db", { _databaseClass: Db });
+    expect(articles).toHaveLength(1);
+    expect(articles[0].title).toBe("来自 ext_json 的标题");
+  });
+
+  it("returns no articles when the table is absent", () => {
+    const Db = makeFakeDb([], { table: "other" });
+    expect(readToutiaoArticles("x.db", { _databaseClass: Db }).articles).toEqual([]);
+  });
+
+  it("buildArticleEvents → BROWSE events, social-toutiao source, stable originalId, read flag", () => {
+    const { events } = buildArticleEvents(
+      [{ groupId: "55", title: "标题", url: "u", category: "headline", behotTime: 2, readTimestamp: 1781700000000, digg: true, repin: false }],
+      { now: 1781800000000 },
+    );
+    expect(events).toHaveLength(1);
+    const e = events[0];
+    expect(e.subtype).toBe("browse");
+    expect(e.source.adapter).toBe("social-toutiao");
+    expect(e.source.originalId).toBe("social-toutiao:article:55");
+    expect(e.occurredAt).toBe(1781700000000); // read_timestamp wins over behot
+    expect(e.extra.kind).toBe("article");
+    expect(e.extra.read).toBe(true);
+    expect(e.extra.digg).toBe(true);
+  });
+
+  it("extractCategory / extractUrl helpers", () => {
+    expect(_internals.extractCategory({ share_url: "x?a=1&category_new=my_tabs_digg&b=2" })).toBe("my_tabs_digg");
+    expect(_internals.extractUrl({ share_info: JSON.stringify({ share_url: "https://h/g/1/?t=1" }) })).toBe("https://h/g/1/");
+  });
+});
+
+describe("articlesToVault — real sqlite + real vault", () => {
+  let dir, dbPath, vdir, vault;
+
+  beforeAll(() => {
+    const Database = require("better-sqlite3-multiple-ciphers");
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "tt-article-"));
+    dbPath = path.join(dir, "news_article.db");
+    const db = new Database(dbPath);
+    db.exec(
+      "CREATE TABLE article (group_id INTEGER, item_id INTEGER, share_info TEXT, ext_json TEXT, " +
+        "share_url TEXT, behot_time INTEGER, read_timestamp INTEGER, is_user_digg INTEGER, is_user_repin INTEGER)",
+    );
+    const ins = db.prepare(
+      "INSERT INTO article (group_id, share_info, share_url, behot_time, read_timestamp, is_user_digg, is_user_repin) VALUES (?,?,?,?,?,?,?)",
+    );
+    ins.run(101, JSON.stringify({ title: "新华视点丨三峡水运新通道 - 今日头条", share_url: "https://m.toutiao.com/g/101/?x=1&category_new=headline" }), "https://m.toutiao.com/g/101/?category_new=headline", 1781700000, 0, 0, 0);
+    ins.run(102, JSON.stringify({ title: "5月汽车出口延续快速增长态势 - 今日头条", share_url: "https://m.toutiao.com/g/102/" }), "https://m.toutiao.com/g/102/?category_new=my_tabs_digg", 1781700100, 1781700200, 1, 0);
+    ins.run(103, "{}", "https://m.toutiao.com/g/103/", 1781700300, 0, 0, 0); // untitled → not ingested
+    db.close();
+
+    vdir = fs.mkdtempSync(path.join(os.tmpdir(), "tt-article-vault-"));
+    vault = new LocalVault({ path: path.join(vdir, "v.db"), key: generateKeyHex() });
+    vault.open();
+  });
+
+  afterAll(() => {
+    try { vault.close(); } catch (_e) { /* best-effort */ }
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
+    try { fs.rmSync(vdir, { recursive: true, force: true }); } catch (_e) { /* best-effort */ }
+  });
+
+  it("reads titled articles and ingests valid BROWSE events", () => {
+    const r = articlesToVault(vault, dbPath, { now: 1781900000000 });
+    expect(r.articles).toBe(2); // the untitled row is skipped
+    expect(r.ingested).toBe(2); // both passed schema validation
+    expect(r.digg).toBe(1);
+    expect(r.read).toBe(1);
+
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const mine = events.filter((e) => e.extra && e.extra.kind === "article");
+    expect(mine.length).toBe(2);
+    expect(mine.every((e) => e.source.adapter === "social-toutiao")).toBe(true);
+  });
+
+  it("re-ingest dedups on the stable per-article originalId", () => {
+    articlesToVault(vault, dbPath, { now: 1781999999999 });
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const mine = events.filter((e) => e.extra && e.extra.kind === "article");
+    expect(mine.length).toBe(2); // still two — updated, not duplicated
+  });
+});

package/__tests__/analysis-skills.test.js

@@ -151,6 +151,43 @@ describe("SpendingSkill", () => {
     expect(r.summary.currency).toBe("CNY");
   });
 
+  it("headline totals come from uncapped sumEventAmount, not the 5000-capped row fetch", async () => {
+    // queryEvents caps at 5000 rows per subtype; a >5000-payment user would
+    // have totalSpend silently undercounted. Fake a vault where the row fetch
+    // returns only a sample but sumEventAmount reports the true sum.
+    const fakeVault = {
+      queryEvents: ({ subtype }) =>
+        subtype === "payment"
+          ? [
+              { id: "p1", subtype: "payment", occurredAt: ts(2026, 5, 1), content: { amount: { value: 10, direction: "out", currency: "CNY" } } },
+              { id: "p2", subtype: "payment", occurredAt: ts(2026, 5, 2), content: { amount: { value: 20, direction: "out", currency: "CNY" } } },
+            ]
+          : [],
+      sumEventAmount: ({ subtype }) =>
+        subtype === "payment"
+          ? { total: 88000, currency: "CNY", count: 5200, byDirection: { out: 88000, in: 1234 }, byCurrency: {} }
+          : { total: 0, currency: "CNY", count: 0, byDirection: { out: 0, in: 0 }, byCurrency: {} },
+    };
+    const r = await new SpendingSkill({ vault: fakeVault }).run({ commentary: false });
+    expect(r.summary.totalSpend).toBe(88000); // true sum, not the 30 from 2 sampled rows
+    expect(r.summary.totalIncome).toBe(1234);
+    expect(r.summary.eventCount).toBe(5200);
+    expect(r.summary.netFlow).toBe(Math.round((1234 - 88000) * 100) / 100);
+  });
+
+  it("with a merchant/person/direction filter it falls back to the row sample (SQL can't express it)", async () => {
+    const fakeVault = {
+      queryEvents: ({ subtype }) =>
+        subtype === "payment"
+          ? [{ id: "p1", subtype: "payment", occurredAt: ts(2026, 5, 1), content: { amount: { value: 30, direction: "out", currency: "CNY" } }, extra: { counterparty: "美团" } }]
+          : [],
+      // would be used by the accurate path — must NOT be when a filter is active
+      sumEventAmount: () => { throw new Error("sumEventAmount must not be called when a row-only filter is set"); },
+    };
+    const r = await new SpendingSkill({ vault: fakeVault }).run({ commentary: false, direction: "out" });
+    expect(r.summary.totalSpend).toBe(30); // from the row sample, not SQL
+  });
+
   it("breakdown by merchant ranks top spenders", async () => {
     setupAlipayPayments();
     const skill = new SpendingSkill({ vault: rig.vault });
@@ -580,12 +617,22 @@ describe("TimelineSkill", () => {
       ingestedAt: Date.now(), source: defaultSource("system-data-android"),
       extra: { kind: "contact-snapshot" },
     });
+    // Aggregate-baseline event (douyin app-usage-profile) — a single rolling
+    // summary, not a discrete activity, so it must be filtered from the timeline.
+    rig.vault.putEvent({
+      id: "event-douyin-usage", type: "event", subtype: "other",
+      occurredAt: ts(2026, 6, 1), actor: "person-self",
+      content: { title: "抖音使用画像：24天/108h" },
+      ingestedAt: Date.now(), source: defaultSource("social-douyin"),
+      extra: { kind: "app-usage-profile" },
+    });
     const skill = new TimelineSkill({ vault: rig.vault });
     const r = await skill.run({ since: ts(2026, 4, 1) });
     const ids = r.entries.map((e) => e.id);
     expect(ids).toContain("act-1");
     expect(ids).not.toContain("event-android-app-com.x");
     expect(ids).not.toContain("event-android-contact-y");
+    expect(ids).not.toContain("event-douyin-usage");
     expect(r.summary.totalEvents).toBe(1);
   });
 });
@@ -645,6 +692,34 @@ describe("OverviewSkill — cross-app unified snapshot", () => {
     expect(r.summary.appsActive).toBe(4); // alipay-bill, shopping-taobao, wechat, social-douyin
   });
 
+  it("byApp/byType/total use uncapped facetCounts, not the row-capped fetch", async () => {
+    // queryEvents hard-caps at 10k rows; on a big vault one dominant app crowds
+    // out the rest, so deriving byApp from the row fetch undercounts (real bug:
+    // social-douyin showed 10 instead of 232). Fake a vault where the capped
+    // row fetch and the SQL GROUP BY disagree, and assert overview trusts SQL.
+    const fakeVault = {
+      facetCounts: () => ({
+        byAdapter: { "social-douyin": 232, "wechat-pc": 100000 },
+        bySubtype: { browse: 232, message: 100000 },
+        byCategory: {},
+        total: 100232,
+        mode: "like",
+        shortQuery: false,
+      }),
+      // simulates the cap: only wechat rows survived the recent-10k window
+      queryEvents: () => [
+        { id: "w1", subtype: "message", occurredAt: ts(2026, 6, 1), actor: "person-self", source: { adapter: "wechat-pc" }, content: {} },
+      ],
+    };
+    const r = await new OverviewSkill({ vault: fakeVault }).run({ commentary: false });
+    const dy = r.byApp.find((a) => a.app === "social-douyin");
+    expect(dy && dy.count).toBe(232); // would be absent/0 if derived from the row fetch
+    expect(r.byApp[0].app).toBe("wechat-pc"); // 100000 sorts first
+    expect(r.summary.totalEvents).toBe(100232);
+    expect(r.summary.appsActive).toBe(2);
+    expect(r.byType.find((t) => t.type === "browse").count).toBe(232);
+  });
+
   it("counts 4 distinct apps + sums cross-app spend + top contact merged", async () => {
     const { vault } = rig;
     makePerson(vault, "p-friend", ["小明"], {}, { adapter: "wechat" });