@chainlesschain/personal-data-hub 0.4.25 → 0.4.26

package/__tests__/sqlite-leaf-salvage.test.js

@@ -0,0 +1,97 @@
+"use strict";
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+// The salvager lives in scripts/ (a standalone forensic tool) but exports its
+// pure parsers for testing.
+const {
+  parseLeafPage,
+  readVarint,
+  serialTypeSize,
+} = require("../../../scripts/android/pdh-sqlite-leaf-salvage.js");
+
+// Build a real (UTF-8) SQLite DB via the SQLCipher-capable driver, then salvage
+// records straight from its raw page bytes — proving the leaf-page parser reads
+// rowids + columns + UTF-8 text correctly (the Method-B reconstruction step for
+// scattered/malformed memory dumps). See docs/internal/pdh-db-decryption-runbook.md.
+describe("pdh-sqlite-leaf-salvage — leaf-page record salvager", () => {
+  let dir, dbPath, buf;
+  beforeAll(() => {
+    const Database = require("better-sqlite3-multiple-ciphers");
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "salvage-"));
+    dbPath = path.join(dir, "u.db");
+    const db = new Database(dbPath);
+    db.exec(
+      "CREATE TABLE msg(msg_uuid TEXT, conversation_id TEXT, sender INTEGER, content TEXT, created_time INTEGER)",
+    );
+    const ins = db.prepare("INSERT INTO msg VALUES(?,?,?,?,?)");
+    ins.run("uuid-1", "conv-1", 111, "你好呀 hello", 1700000000000);
+    ins.run("uuid-2", "conv-1", 222, "在吗？晚上一起吃饭", 1700000001000);
+    ins.run("uuid-3", "conv-2", 333, "ok 👍", 1700000002000);
+    db.close();
+    buf = fs.readFileSync(dbPath);
+  });
+  afterAll(() => {
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) { /* ignore */ }
+  });
+
+  it("varint + serialTypeSize basics", () => {
+    expect(readVarint(Buffer.from([0x01]), 0)[0]).toBe(1n);
+    expect(readVarint(Buffer.from([0x81, 0x00]), 0)[0]).toBe(128n);
+    expect(serialTypeSize(1n)).toBe(1); // 1-byte int
+    expect(serialTypeSize(6n)).toBe(8); // 8-byte int
+    expect(serialTypeSize(13n)).toBe(0); // text len 0
+    expect(serialTypeSize(0x29n)).toBe(14); // text serial 41 → 14 bytes
+  });
+
+  it("salvages all 3 msg rows with correct columns + UTF-8 text", () => {
+    const PAGE = 4096;
+    const records = [];
+    for (let base = 0; base + PAGE <= buf.length; base += PAGE) {
+      const recs = parseLeafPage(buf, base, PAGE, 3);
+      if (recs) records.push(...recs);
+    }
+    // find the msg rows (5 cols, content is the 4th)
+    const msgRows = records.filter((r) => r.cols.length === 5);
+    expect(msgRows.length).toBe(3);
+    const byUuid = Object.fromEntries(msgRows.map((r) => [r.cols[0], r]));
+    expect(byUuid["uuid-1"].cols[2]).toBe(111); // sender int
+    expect(byUuid["uuid-1"].cols[3]).toBe("你好呀 hello"); // UTF-8 intact
+    expect(byUuid["uuid-2"].cols[3]).toBe("在吗？晚上一起吃饭");
+    expect(byUuid["uuid-2"].cols[4]).toBe(1700000001000); // created_time
+    expect(byUuid["uuid-3"].cols[3]).toBe("ok 👍"); // emoji (4-byte UTF-8)
+  });
+
+  it("finds a leaf page at a NON-4096-aligned offset (unaligned scan)", () => {
+    const PAGE = 4096;
+    // locate a real data leaf page on the aligned grid
+    let leaf = null;
+    for (let base = 0; base + PAGE <= buf.length; base += PAGE) {
+      const recs = parseLeafPage(buf, base, PAGE, 3);
+      if (recs && recs.some((r) => r.cols.length === 5)) { leaf = buf.slice(base, base + PAGE); break; }
+    }
+    expect(leaf).not.toBeNull();
+    // embed it at a 512-aligned-but-not-4096-aligned offset inside a zero buffer
+    const big = Buffer.alloc(PAGE * 4, 0);
+    const off = 512 * 3; // 1536: hit by stride-512, missed by stride-4096
+    leaf.copy(big, off);
+    // aligned 4096-grid misses it
+    expect(parseLeafPage(big, 0, PAGE, 3)).toBeNull();
+    expect(parseLeafPage(big, PAGE, PAGE, 3)).toBeNull();
+    // unaligned stride finds it at its true offset
+    const recs = parseLeafPage(big, off, PAGE, 3);
+    expect(recs).not.toBeNull();
+    expect(recs.some((r) => r.cols[0] === "uuid-1")).toBe(true);
+  });
+
+  it("returns null for non-leaf / garbage pages", () => {
+    const garbage = Buffer.alloc(4096, 0xff);
+    expect(parseLeafPage(garbage, 0, 4096, 3)).toBeNull();
+    const zeros = Buffer.alloc(4096, 0);
+    expect(parseLeafPage(zeros, 0, 4096, 3)).toBeNull();
+  });
+});

package/lib/adapters/social-douyin/index.js

@@ -63,7 +63,8 @@ const KIND_FAVOURITE = "favourite"; // v0.3 (X-Bogus required)
 const KIND_LIKE = "like";         // v0.3 (X-Bogus required)
 const KIND_SEARCH = "search";     // legacy sqlite-mode only
 const KIND_MESSAGE = "message";   // Phase 2a — IM private messages from <uid>_im.db (abrignoni DFIR)
-const KIND_CONTACT = "contact";   // Phase 2a — SIMPLE_USER table contacts/follows from <uid>_im.db
+const KIND_CONTACT = "contact";   // Phase 2a — SIMPLE_USER/participant contacts from <uid>_im.db
+const KIND_CONVERSATION = "conversation"; // device-verified — conversation_list thread → TOPIC
 
 // Forward-compat: list every kind v0.3+ may emit so cc adapter accepts
 // snapshots from a newer Android even if this JS hasn't been bumped yet.
@@ -258,7 +259,7 @@ class DouyinAdapter {
     if (Number.isInteger(opts.limitContacts)) parseOpts.limitContacts = opts.limitContacts;
     if (this._deps.dbDriverFactory) parseOpts._databaseClass = this._deps.dbDriverFactory();
 
-    const { messages, contacts, diagnostic } = parseImDb(dbPath, parseOpts);
+    const { messages, contacts, conversations, diagnostic } = parseImDb(dbPath, parseOpts);
     if (typeof opts.onProgress === "function") {
       try {
         opts.onProgress({ phase: "im-db-parsed", adapter: NAME, ...diagnostic });
@@ -314,6 +315,27 @@ class DouyinAdapter {
         emitted += 1;
       }
     }
+
+    if (include[KIND_CONVERSATION] !== false) {
+      for (const cv of conversations || []) {
+        if (emitted >= limit) return;
+        if (!cv || typeof cv !== "object" || !cv.conversationId) continue;
+        yield {
+          adapter: NAME,
+          kind: KIND_CONVERSATION,
+          originalId: stableOriginalId(
+            KIND_CONVERSATION,
+            `conv-${cv.conversationId}`,
+          ),
+          capturedAt:
+            typeof cv.lastMsgTimeMs === "number" && cv.lastMsgTimeMs > 0
+              ? cv.lastMsgTimeMs
+              : fallbackCapturedAt,
+          payload: { kind: KIND_CONVERSATION, ...cv },
+        };
+        emitted += 1;
+      }
+    }
   }
 
   async *_syncViaSnapshot(opts) {
@@ -454,6 +476,9 @@ class DouyinAdapter {
     if (kind === KIND_CONTACT) {
       return normalizeContact(p, raw, ingestedAt);
     }
+    if (kind === KIND_CONVERSATION) {
+      return normalizeConversation(p, raw, ingestedAt);
+    }
     throw new Error(`DouyinAdapter.normalize: unknown kind ${kind}`);
   }
 }
@@ -697,6 +722,35 @@ function normalizeContact(p, raw, ingestedAt) {
   };
 }
 
+function normalizeConversation(p, raw, ingestedAt) {
+  // conversation_list row from <uid>_im.db → a TOPIC (one chat thread).
+  const convId =
+    (typeof p.conversationId === "string" && p.conversationId) ||
+    (typeof p.conversationId === "number" && String(p.conversationId)) ||
+    null;
+  const occurredAt = raw.capturedAt || ingestedAt;
+  const source = buildSource(raw, occurredAt, CAPTURED_BY.SQLITE);
+  return {
+    events: [], persons: [], places: [], items: [],
+    topics: [{
+      id: convId ? `topic-douyin-conv-${convId}` : `topic-douyin-conv-${newId()}`,
+      type: ENTITY_TYPES.TOPIC,
+      name: convId ? `抖音会话 ${convId}` : "抖音会话",
+      ingestedAt,
+      source,
+      extra: {
+        platform: "douyin",
+        conversationId: convId,
+        conversationType:
+          typeof p.conversationType === "number" ? p.conversationType : null,
+        lastMsgTimeMs:
+          typeof p.lastMsgTimeMs === "number" ? p.lastMsgTimeMs : null,
+        stranger: typeof p.stranger === "boolean" ? p.stranger : null,
+      },
+    }],
+  };
+}
+
 module.exports = {
   DouyinAdapter,
   NAME,

package/lib/adapters/social-douyin-adb/collector.js

@@ -25,6 +25,8 @@ const {
   writeSnapshotJson,
   cleanupSnapshotJson,
 } = require("./snapshot-builder");
+const { salvageFile } = require("../../forensics/leaf-salvage");
+const { mapMsgRecords, inferMsgColumns } = require("./salvage-mapper");
 
 /**
  * Pull IM db → parse → write snapshot. Returns the staging path + counts
@@ -159,6 +161,102 @@ async function collectAndSync(bridge, registry, opts = {}) {
   };
 }
 
+// ── Salvage path (Method B /proc/mem dump → leaf-salvage → snapshot) ──────
+//
+// The key-free decryption breakthrough: a rooted device dumps a running app's
+// decrypted SQLite pages from /proc/<pid>/mem, then this salvages the message
+// records straight out of the leaf pages (no key, no password) and ingests them
+// through the same social-douyin snapshot path. Closes the loop: dump → salvage
+// → THIS → PDH entities. See docs/internal/pdh-db-decryption-runbook.md §3.5.
+
+/**
+ * Salvage records from a memory dump → social-douyin snapshot JSON.
+ *
+ * @param {string} dumpPath  path to the /proc/mem dump (or concatenated dumps)
+ * @param {{
+ *   uid?: string,             // account uid; defaults to "salvage" placeholder
+ *   columns?: string[],       // explicit msg column order; else inferMsgColumns
+ *   pageSize?: number, minCols?: number, unaligned?: boolean, stride?: number,
+ *   displayName?: string,
+ *   stagingDir?: string,
+ *   now?: () => number,
+ * }} [opts]
+ * @returns {{snapshotPath: string, uid: string, eventCounts: object, salvage: object}}
+ */
+function salvageDumpToSnapshot(dumpPath, opts = {}) {
+  if (typeof dumpPath !== "string" || dumpPath.length === 0) {
+    throw new TypeError("salvageDumpToSnapshot: dumpPath must be a non-empty string");
+  }
+  const now = opts.now || Date.now;
+  const { records, pages } = salvageFile(dumpPath, {
+    pageSize: opts.pageSize,
+    minCols: opts.minCols,
+    unaligned: opts.unaligned,
+    stride: opts.stride,
+  });
+  // Leaf pages carry no column names — use the caller's explicit order when
+  // known (most accurate), else heuristically infer content/created_time.
+  const columns = Array.isArray(opts.columns) && opts.columns.length
+    ? opts.columns
+    : inferMsgColumns(records);
+  const messages = mapMsgRecords(records, columns);
+  const uid = typeof opts.uid === "string" && opts.uid.length ? opts.uid : "salvage";
+  const snapshot = buildSnapshot({
+    uid,
+    displayName: opts.displayName,
+    messages,
+    contacts: [],
+    snapshottedAt: now(),
+  });
+  const snapshotPath = writeSnapshotJson(snapshot, { dir: opts.stagingDir });
+  return {
+    snapshotPath,
+    uid,
+    eventCounts: { message: messages.length, total: messages.length },
+    salvage: { leafPages: pages, recordsSalvaged: records.length, columns },
+  };
+}
+
+/**
+ * One-shot: salvage dump → snapshot → syncAdapter("social-douyin") → cleanup.
+ *
+ * @param {object} registry  AdapterRegistry (must expose syncAdapter)
+ * @param {string} dumpPath
+ * @param {object} [opts]    forwarded to salvageDumpToSnapshot
+ * @returns {Promise<object>} SyncReport + salvage diagnostic
+ */
+async function salvageAndSync(registry, dumpPath, opts = {}) {
+  if (!registry || typeof registry.syncAdapter !== "function") {
+    throw new TypeError(
+      "salvageAndSync: registry must expose syncAdapter(name, options)",
+    );
+  }
+  const res = salvageDumpToSnapshot(dumpPath, opts);
+  let syncReport = null;
+  let cleanupFailed = false;
+  try {
+    syncReport = await registry.syncAdapter("social-douyin", {
+      inputPath: res.snapshotPath,
+    });
+  } finally {
+    try {
+      cleanupSnapshotJson(res.snapshotPath);
+    } catch (_e) {
+      cleanupFailed = true;
+    }
+  }
+  return {
+    ...syncReport,
+    douyin: {
+      uid: res.uid,
+      eventCounts: res.eventCounts,
+      salvage: res.salvage,
+      mode: "salvage",
+      cleanupFailed,
+    },
+  };
+}
+
 // ── Watch-history (video_record.db) path ─────────────────────────────────
 // Distinct from the IM-db path above: pulls the plaintext video_record.db and
 // emits `history` events (KIND_HISTORY → BROWSE) the social-douyin adapter
@@ -276,4 +374,6 @@ module.exports = {
   collectAndSync,
   collectWatchHistory,
   collectWatchHistoryAndSync,
+  salvageDumpToSnapshot,
+  salvageAndSync,
 };

package/lib/adapters/social-douyin-adb/im-db-parser.js

@@ -130,14 +130,22 @@ function parseImDb(dbPath, opts = {}) {
       : 5_000;
   const Database = opts._databaseClass || loadDatabaseClass();
   const db = new Database(dbPath, { readonly: true });
+  const limitConversations =
+    Number.isInteger(opts.limitConversations) && opts.limitConversations > 0
+      ? opts.limitConversations
+      : 5_000;
   const out = {
     messages: [],
     contacts: [],
+    conversations: [],
     diagnostic: {
       messageCount: 0,
       contactCount: 0,
+      conversationCount: 0,
       hadMsgTable: false,
       hadSimpleUserTable: false,
+      hadParticipantTable: false,
+      hadConversationListTable: false,
     },
   };
   try {
@@ -230,6 +238,83 @@ function parseImDb(dbPath, opts = {}) {
         out.diagnostic.contactCount = out.contacts.length;
       }
     }
+
+    // ─── participant table (device-verified 2026-06-16) ──────────────────
+    // Real Douyin IM schema keeps conversation members in `participant`
+    // (conversation_id, user_id, sort_order; UNIQUE(conversation_id,user_id)),
+    // NOT SIMPLE_USER (which is older/other builds). Pull distinct member uids
+    // as contacts — uid-only (nickname/avatar live in a separate user table),
+    // so a PERSON gets created keyed by douyin-uid even without a name.
+    // Dedup against contacts already harvested from SIMPLE_USER.
+    const partTableInfo = trySelect(db, "PRAGMA table_info(participant)");
+    if (Array.isArray(partTableInfo) && partTableInfo.length > 0) {
+      out.diagnostic.hadParticipantTable = true;
+      const columns = new Set(partTableInfo.map((r) => r.name));
+      const uidCol = pickCol(columns, ["user_id", "uid", "UID"]);
+      if (uidCol) {
+        const seen = new Set(
+          out.contacts.map((c) => c.uid).filter(Boolean),
+        );
+        const sql =
+          `SELECT DISTINCT ${uidCol} AS uid FROM participant ` +
+          `WHERE ${uidCol} IS NOT NULL LIMIT ${limitContacts}`;
+        const rows = trySelect(db, sql) || [];
+        for (const r of rows) {
+          const uid = r.uid != null ? String(r.uid) : null;
+          if (!uid || seen.has(uid)) continue;
+          seen.add(uid);
+          out.contacts.push({
+            uid,
+            shortId: null,
+            name: null,
+            avatarUrl: null,
+            followStatus: null,
+            fromParticipant: true,
+          });
+        }
+        out.diagnostic.contactCount = out.contacts.length;
+      }
+    }
+
+    // ─── conversation_list table (device-verified 2026-06-16) ────────────
+    // Each row is a chat thread → PDH TOPIC. Columns vary by build; pick
+    // defensively. conversation_id is the only hard requirement.
+    const convTableInfo = trySelect(db, "PRAGMA table_info(conversation_list)");
+    if (Array.isArray(convTableInfo) && convTableInfo.length > 0) {
+      out.diagnostic.hadConversationListTable = true;
+      const columns = new Set(convTableInfo.map((r) => r.name));
+      const idCol = pickCol(columns, ["conversation_id", "conv_id", "id"]);
+      const typeCol = pickCol(columns, ["type", "conversation_type", "conv_type"]);
+      const lastTimeCol = pickCol(columns, [
+        "last_msg_create_time",
+        "last_message_time",
+        "updated_time",
+      ]);
+      const strangerCol = pickCol(columns, ["stranger", "is_stranger"]);
+      if (idCol) {
+        const fields = [`${idCol} AS convId`];
+        if (typeCol) fields.push(`${typeCol} AS convType`);
+        if (lastTimeCol) fields.push(`${lastTimeCol} AS lastMsgTime`);
+        if (strangerCol) fields.push(`${strangerCol} AS stranger`);
+        const orderBy = lastTimeCol ? ` ORDER BY ${lastTimeCol} DESC` : "";
+        const sql =
+          `SELECT ${fields.join(", ")} FROM conversation_list` +
+          `${orderBy} LIMIT ${limitConversations}`;
+        const rows = trySelect(db, sql) || [];
+        for (const r of rows) {
+          if (r.convId == null) continue;
+          out.conversations.push({
+            conversationId: String(r.convId),
+            conversationType:
+              typeof r.convType === "number" ? r.convType : null,
+            lastMsgTimeMs: normalizeEpochMs(r.lastMsgTime),
+            stranger:
+              typeof r.stranger === "number" ? r.stranger === 1 : null,
+          });
+        }
+        out.diagnostic.conversationCount = out.conversations.length;
+      }
+    }
   } finally {
     db.close();
   }

package/lib/adapters/social-douyin-adb/index.js

@@ -43,6 +43,8 @@ const {
   collectAndSync,
   collectWatchHistory,
   collectWatchHistoryAndSync,
+  salvageDumpToSnapshot,
+  salvageAndSync,
 } = require("./collector");
 const {
   createDouyinWatchExtension,
@@ -71,4 +73,7 @@ module.exports = {
   // Collector orchestrator
   collect,
   collectAndSync,
+  // Method B salvage path (/proc/mem dump → leaf-salvage → snapshot → ingest)
+  salvageDumpToSnapshot,
+  salvageAndSync,
 };

package/lib/adapters/social-douyin-adb/salvage-mapper.js

@@ -0,0 +1,119 @@
+"use strict";
+/*
+ * Glue: leaf-salvaged records → parseImDb-shaped output.
+ *
+ * The leaf-page salvager (scripts/android/pdh-sqlite-leaf-salvage.js) emits raw
+ * positional tuples {rowid, cols:[...]} (leaf pages carry no column names). This
+ * maps them into the SAME shape `parseImDb` returns ({messages, contacts,
+ * conversations}) so the existing DouyinAdapter.normalize path ingests them
+ * unchanged — closing the loop: Method-B dump → salvage → THIS → PDH entities.
+ *
+ * Column order comes from the table's CREATE TABLE (see docs/internal/
+ * pdh-app-db-schemas.md or grep the dump). Pass it explicitly for correctness;
+ * `inferMsgColumns` offers a heuristic fallback (content=JSON/longest text,
+ * created_time=epoch int) when the exact order is unknown.
+ */
+const { _internals } = require("./im-db-parser");
+const { extractTextFromContent, normalizeEpochMs } = _internals;
+
+function zip(cols, names) {
+  const o = {};
+  for (let i = 0; i < names.length; i++) o[names[i]] = cols[i];
+  return o;
+}
+
+// Map msg-table salvaged records given the ordered column names.
+function mapMsgRecords(records, columns) {
+  const out = [];
+  for (const r of records || []) {
+    if (!r || !Array.isArray(r.cols)) continue;
+    const row = zip(r.cols, columns);
+    if (row.content == null && row.created_time == null) continue;
+    const t = typeof row.created_time === "number" ? row.created_time
+      : Number(row.created_time);
+    out.push({
+      senderUid: row.sender != null ? String(row.sender) : null,
+      conversationId: row.conversation_id != null ? String(row.conversation_id) : null,
+      createdTimeMs: normalizeEpochMs(Number.isFinite(t) ? t : 0),
+      text: extractTextFromContent(row.content),
+      readStatus: typeof row.read_status === "number" ? row.read_status : null,
+      contentBlob: typeof row.content === "string" ? row.content : null,
+    });
+  }
+  return out;
+}
+
+function mapParticipantRecords(records, columns) {
+  const seen = new Set();
+  const out = [];
+  for (const r of records || []) {
+    if (!r || !Array.isArray(r.cols)) continue;
+    const row = zip(r.cols, columns);
+    const uid = row.user_id != null ? String(row.user_id) : null;
+    if (!uid || seen.has(uid)) continue;
+    seen.add(uid);
+    out.push({ uid, shortId: null, name: null, avatarUrl: null, followStatus: null, fromParticipant: true });
+  }
+  return out;
+}
+
+function mapConversationRecords(records, columns) {
+  const out = [];
+  for (const r of records || []) {
+    if (!r || !Array.isArray(r.cols)) continue;
+    const row = zip(r.cols, columns);
+    if (row.conversation_id == null) continue;
+    out.push({
+      conversationId: String(row.conversation_id),
+      conversationType: typeof row.type === "number" ? row.type : null,
+      lastMsgTimeMs: normalizeEpochMs(Number(row.last_msg_create_time) || 0),
+      stranger: typeof row.stranger === "number" ? row.stranger === 1 : null,
+    });
+  }
+  return out;
+}
+
+/**
+ * Heuristic: when the exact `msg` column order is unknown, guess content +
+ * created_time positions from value shapes (content = a JSON-ish / longest
+ * string; created_time = the largest plausible-epoch integer). Returns a column
+ * name array usable with mapMsgRecords (unknown slots get c0,c1,...).
+ */
+function inferMsgColumns(records) {
+  const sample = (records || []).find((r) => r && Array.isArray(r.cols) && r.cols.length >= 3);
+  if (!sample) return [];
+  const cols = sample.cols;
+  const names = cols.map((_, i) => `c${i}`);
+  let contentIdx = -1, contentScore = -1;
+  let timeIdx = -1, timeVal = -1;
+  for (let i = 0; i < cols.length; i++) {
+    const v = cols[i];
+    if (typeof v === "string") {
+      const score = (v.trim().startsWith("{") ? 1e6 : 0) + v.length;
+      if (score > contentScore) { contentScore = score; contentIdx = i; }
+    } else if (typeof v === "number" && v > 1e9 && v > timeVal) {
+      // largest epoch-ish int → created_time (ms/sec/us all > 1e9)
+      timeVal = v; timeIdx = i;
+    }
+  }
+  if (contentIdx >= 0) names[contentIdx] = "content";
+  if (timeIdx >= 0 && timeIdx !== contentIdx) names[timeIdx] = "created_time";
+  return names;
+}
+
+// One-shot: salvaged records (mixed) → parseImDb shape, given per-table columns.
+function mapSalvaged({ msg, participant, conversation } = {}) {
+  return {
+    messages: msg ? mapMsgRecords(msg.records, msg.columns) : [],
+    contacts: participant ? mapParticipantRecords(participant.records, participant.columns) : [],
+    conversations: conversation ? mapConversationRecords(conversation.records, conversation.columns) : [],
+  };
+}
+
+module.exports = {
+  mapMsgRecords,
+  mapParticipantRecords,
+  mapConversationRecords,
+  inferMsgColumns,
+  mapSalvaged,
+};