@chainlesschain/personal-data-hub 0.4.30 → 0.4.32

package/lib/forensics/plaintext-db-collect.js

@@ -0,0 +1,89 @@
+'use strict';
+/**
+ * plaintext-db-collect — generic ingester for an app's PLAINTEXT SQLite dbs.
+ *
+ * Many app dbs aren't encrypted (browse/read/history/content/config). The Magisk
+ * daemon (root) can read them directly on MIUI; this turns any such db into vault
+ * events by pulling readable text records from each table — comprehensive coverage
+ * of the "明文库" personal data without per-db schema work. Encrypted IM dbs
+ * (QQNT/WeChat/WCDB2) have their own decrypt collectors; this is for the rest.
+ *
+ * Heuristics: skip system/Room-internal tables; per table, take TEXT-ish columns;
+ * keep a row only if it has a meaningful readable value (CJK, or ≥6 letters, not a
+ * uuid/hash/base64 blob); derive a time from any *time/*date/created/_at column.
+ * Pure Node + a caller-provided better-sqlite3 ctor.
+ */
+const crypto = require('crypto');
+
+const SKIP_TABLE = /^(android_metadata|sqlite_|room_master_table|_room|.*_fts(_.*)?$|.*_log$|.*_index$)/i;
+const TEXT_TYPE = /text|char|clob|json|varchar|string/i;
+const TIME_COL = /(^|_)(time|date|created|updated|_at|timestamp|ctime|mtime)($|_)/i;
+const NOISE_VAL = /^(([0-9a-f]{16,})|([A-Za-z0-9+/=_-]{24,})|(\d{6,})|(https?:\/\/\S+)|(\{.{0,3}\})|(\[\]))$/;
+
+function normTime(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n <= 0) return 0;
+  if (n > 1e16) return Math.floor(n / 1e6); // ns? → ms
+  if (n > 1e14) return Math.floor(n / 1e3); // µs → ms
+  if (n > 1e12) return n; // ms
+  if (n > 1e9) return n * 1000; // s → ms
+  return 0;
+}
+function readable(v) {
+  if (typeof v !== 'string') return false;
+  const s = v.trim();
+  if (s.length < 4 || s.length > 2000) return false;
+  if (NOISE_VAL.test(s)) return false;
+  if (/[一-鿿]/.test(s)) return true; // any CJK
+  return (s.match(/[A-Za-z]/g) || []).length >= 6; // or enough letters
+}
+
+/**
+ * @param Database better-sqlite3 ctor (injected)
+ * @param dbPath   path to a plaintext SQLite db
+ * @param app      app key (→ source.adapter `local-<app>`)
+ * @returns {Array} vault events (subtype:"record")
+ */
+function ingestPlaintextDb(Database, dbPath, app) {
+  const db = new Database(dbPath, { readonly: true });
+  const events = [];
+  const dbName = String(dbPath).split(/[\\/]/).pop();
+  try {
+    const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all()
+      .map((r) => r.name).filter((t) => !SKIP_TABLE.test(t));
+    for (const t of tables) {
+      let cols, rows;
+      try {
+        cols = db.prepare(`PRAGMA table_info("${t}")`).all();
+        rows = db.prepare(`SELECT * FROM "${t}" LIMIT 2000`).all();
+      } catch { continue; }
+      const textCols = cols.filter((c) => TEXT_TYPE.test(c.type || '')).map((c) => c.name);
+      if (!textCols.length || !rows.length) continue;
+      const timeCol = (cols.find((c) => TIME_COL.test(c.name)) || {}).name;
+      for (const row of rows) {
+        const parts = textCols.map((c) => row[c]).filter(readable);
+        if (!parts.length) continue;
+        const text = [...new Set(parts)].join(' · ').slice(0, 800);
+        const occurredAt = timeCol ? normTime(row[timeCol]) : 0;
+        const id = `${app}:${dbName}:${t}:` +
+          crypto.createHash('md5').update(JSON.stringify(row)).digest('hex').slice(0, 12);
+        events.push({
+          type: 'event', subtype: 'record',
+          id, occurredAt: occurredAt || Date.now(),
+          content: { text },
+          source: {
+            adapter: `local-${app}`, adapterVersion: '0.1.0',
+            originalId: `${dbName}:${t}`, capturedAt: occurredAt || Date.now(), capturedBy: 'sqlite',
+          },
+          topics: [`local-${app}`, `db-${dbName}`],
+          ingestedAt: Date.now(),
+        });
+      }
+    }
+  } finally {
+    db.close();
+  }
+  return events;
+}
+
+module.exports = { ingestPlaintextDb, readable, normTime };

package/lib/forensics/qq-nt-collect.js

@@ -0,0 +1,190 @@
+'use strict';
+/**
+ * qq-nt-collect — on-device-ready QQNT (nt_msg.db) decrypt + protobuf-parse.
+ *
+ * Pure Node (crypto + a caller-provided better-sqlite3 ctor) — NO frida, NO adb.
+ * This is the bundle-shipped core behind `cc hub collect-qq`, which the Android
+ * `CollectQqNativeTool` invokes after su-staging the encrypted DB + uid list. The
+ * exact same logic runs on PC (USB pull) and on-device (app su-read).
+ *
+ * Method (key DERIVED, no frida): `key = MD5(MD5(uid) + rand)` → SQLCipher
+ * PBKDF2-HMAC-SHA512/1 (iter 4000) → AES-256-CBC. `rand` is read from the 1024-byte
+ * plaintext header; `uid` (QQNT `u_...`) is brute-forced from candidates — page 1
+ * decrypting to a valid SQLite header identifies the self uid. (See
+ * scripts/android/pdh-qq-android-decrypt.mjs for the standalone PC tool this was
+ * extracted from; behaviour is byte-identical.)
+ */
+const crypto = require('crypto');
+const md5 = (s) => crypto.createHash('md5').update(s).digest('hex');
+
+/** Read `rand` (protobuf field after "QQ_NT DB") from the plaintext header. */
+function extractRand(raw) {
+  const head = raw.subarray(0, 256).toString('latin1');
+  const i = head.indexOf('QQ_NT DB');
+  if (i < 0) return null;
+  const m = head.slice(i + 8, i + 40).match(/[A-Za-z0-9]{6,12}/);
+  return m ? m[0] : null;
+}
+
+/** Detect HMAC algo named in the header (informational). */
+function headerHmac(raw) {
+  return (/HMAC_SHA\d+/.exec(raw.subarray(0, 256).toString('latin1')) || [])[0] || null;
+}
+
+// decrypt one page slice. p1=true → page 1 (first 16 bytes = salt, skipped).
+function decPage(page, encKey, reserve, ivOff, p1) {
+  const ro = page.length - reserve;
+  const ct = page.subarray(p1 ? 16 : 0, ro);
+  const iv = page.subarray(ro + ivOff, ro + ivOff + 16);
+  if (iv.length < 16) return null;
+  try {
+    const d = crypto.createDecipheriv('aes-256-cbc', encKey, iv);
+    d.setAutoPadding(false);
+    return Buffer.concat([d.update(ct), d.final()]);
+  } catch {
+    return null;
+  }
+}
+const validHdr = (pt) => pt && pt.length > 8 && pt[5] === 64 && pt[6] === 32 && pt[7] === 32;
+const CFG = [[4096, 48, 0], [4096, 80, 0], [4096, 48, 20]];
+
+/**
+ * Brute the key over uid candidates, then decrypt every page.
+ * @returns {{decrypted: Buffer, uid: string, kdf: string}|null} null = no uid matched.
+ */
+function deriveAndDecrypt(raw, uids, rand) {
+  if (!raw || raw.length <= 1024 || !rand || !uids || !uids.length) return null;
+  const body = raw.subarray(1024);
+  const salt = body.subarray(0, 16);
+  let hit = null;
+  for (const uid of uids) {
+    const pass = md5(md5(uid) + rand);
+    for (const algo of ['sha512', 'sha1']) {
+      const ek = crypto.pbkdf2Sync(Buffer.from(pass, 'utf8'), salt, 4000, 32, algo);
+      for (const [pg, rs, iv] of CFG) {
+        if (validHdr(decPage(body.subarray(0, pg), ek, rs, iv, true))) {
+          hit = { uid, algo, page: pg, reserve: rs, ivOff: iv, ek };
+          break;
+        }
+      }
+      if (hit) break;
+    }
+    if (hit) break;
+  }
+  if (!hit) return null;
+  const { ek, page, reserve, ivOff } = hit;
+  const n = Math.floor(body.length / page);
+  const out = [];
+  for (let i = 0; i < n; i++) {
+    const pt = decPage(body.subarray(i * page, (i + 1) * page), ek, reserve, ivOff, i === 0);
+    const full = Buffer.alloc(page);
+    if (i === 0) {
+      Buffer.from('SQLite format 3\0').copy(full, 0);
+      if (pt) pt.copy(full, 16);
+    } else if (pt) {
+      pt.copy(full, 0);
+    }
+    out.push(full);
+  }
+  return { decrypted: Buffer.concat(out), uid: hit.uid, kdf: hit.algo };
+}
+
+// ── protobuf message-body → readable text (from pdh-qq-ingest.mjs) ──────────
+function readVarint(buf, p) {
+  let shift = 0, r = 0n;
+  while (p < buf.length) {
+    const b = buf[p++];
+    r |= BigInt(b & 0x7f) << BigInt(shift);
+    if (!(b & 0x80)) break;
+    shift += 7;
+  }
+  return [r, p];
+}
+function* fields(buf) {
+  let p = 0;
+  while (p < buf.length) {
+    let tag;[tag, p] = readVarint(buf, p); tag = Number(tag);
+    const wire = tag & 7;
+    if (wire === 0) { let v;[v, p] = readVarint(buf, p); } else if (wire === 2) {
+      let len;[len, p] = readVarint(buf, p); len = Number(len);
+      const data = buf.subarray(p, p + len); p += len; yield data;
+    } else if (wire === 5) p += 4; else if (wire === 1) p += 8; else return;
+  }
+}
+const readable = (s) => s && /[一-鿿　-ヿA-Za-z0-9]/.test(s) && !/[�]/.test(s) &&
+  [...s].every((c) => c.charCodeAt(0) >= 32 || c === '\n');
+function extractTexts(buf, depth, out) {
+  if (depth > 6 || !buf || buf.length === 0) return;
+  for (const data of fields(buf)) {
+    if (!data || data.length === 0) continue;
+    let s = null; try { s = data.toString('utf8'); } catch {}
+    if (s && Buffer.from(s, 'utf8').equals(data) && readable(s) && s.length <= 1000 &&
+      !/^https?:\/\/\S+$/.test(s)) out.push(s);
+    if (data.length >= 2) extractTexts(data, depth + 1, out);
+  }
+}
+function bodyText(blob) {
+  if (!Buffer.isBuffer(blob)) return null;
+  const out = []; extractTexts(blob, 0, out);
+  const cjk = out.filter((t) => /[一-鿿]/.test(t)).sort((a, b) => b.length - a.length);
+  return cjk[0] || out.sort((a, b) => b.length - a.length)[0] || null;
+}
+
+/**
+ * Parse a DECRYPTED nt_msg.db into vault events (qq-pc adapter shape).
+ * @param Database  a better-sqlite3 constructor (injected by the caller/bundle)
+ * @param dbPath    path to the decrypted nt_msg.db
+ * @param self      the user's own QQ number (attribution fallback)
+ * @returns {Array} event objects ready for vault.putEvent
+ */
+function parseEvents(Database, dbPath, self) {
+  const src = new Database(dbPath, { readonly: true });
+  const events = [];
+  const num = (v) => (typeof v === 'bigint' ? Number(v) : v);
+  const ingestTable = (table, isGroup) => {
+    let rows;
+    try {
+      rows = src.prepare(
+        `SELECT [40001] msgId,[40020] uid,[40011] type,[40033] sender,[40021] peer,` +
+        `[40050] t,[40800] body FROM ${table}`,
+      ).safeIntegers().all();
+    } catch { return; }
+    for (const r of rows) {
+      const type = num(r.type);
+      if (type === 5 || type === 9) continue; // grey-bar / system
+      const text = bodyText(r.body);
+      if (!text || !/[一-鿿A-Za-z0-9]/.test(text)) continue;
+      // QQ service/gray-tip config messages (e.g. com.tencent.* push configs) —
+      // not real chat; drop so the vault holds conversation, not service noise.
+      if (text.includes('com.tencent.') || text.includes('public desc')) continue;
+      const msgId = typeof r.msgId === 'bigint' ? r.msgId.toString() : String(r.msgId);
+      const sender = String(num(r.sender) || '');
+      const peer = String(num(r.peer) || '');
+      const occurredAt = num(r.t) * 1000;
+      if (!occurredAt) continue;
+      const actor = sender ? `person-qq-${sender}` : `person-qq-${self}`;
+      const participants = [actor];
+      participants.push(isGroup ? `group-qq-${peer}` : `person-qq-${peer}`);
+      events.push({
+        type: 'event', subtype: 'message', id: `qq:${table}:${msgId}`,
+        occurredAt, actor, participants,
+        content: { text: isGroup ? `[群${peer}] ${text}` : text },
+        topics: isGroup ? [`group-qq-${peer}`] : undefined,
+        source: {
+          adapter: 'qq-pc', adapterVersion: '0.1.0', originalId: `${table}:${msgId}`,
+          capturedAt: occurredAt, capturedBy: 'sqlite',
+        },
+        ingestedAt: Date.now(),
+      });
+    }
+  };
+  try {
+    ingestTable('c2c_msg_table', false);
+    ingestTable('group_msg_table', true);
+  } finally {
+    src.close();
+  }
+  return events;
+}
+
+module.exports = { extractRand, headerHmac, deriveAndDecrypt, bodyText, parseEvents };

package/lib/forensics/wechat-collect.js

@@ -0,0 +1,134 @@
+'use strict';
+/**
+ * wechat-collect — on-device-ready WeChat (EnMicroMsg.db) decrypt + parse.
+ *
+ * Pure Node (crypto + a caller-provided better-sqlite3 ctor) — NO frida. Same
+ * model as qq-nt-collect.js: derived key (`MD5(IMEI+uin)[:7]`) → SQLCipher
+ * (page-level AES-256-CBC) → parse message/rcontact/chatroom → vault events.
+ * The bundle-shipped core behind `cc hub collect-wechat`; the Magisk daemon
+ * stages EnMicroMsg.db + uins (+ IMEI candidates) and this decrypts on-device.
+ * Extracted from scripts/android/pdh-wechat-decrypt.mjs (byte-identical crypto;
+ * verified WeChat 8.0.74 on chopin).
+ *
+ * IMEI on Android 13 is often unreadable → pass a SAVED 7-char key (once found
+ * it never changes for an account) or a frida raw key as a fallback candidate.
+ */
+const crypto = require('crypto');
+const md5 = (s) => crypto.createHash('md5').update(s).digest('hex');
+
+const CONFIGS = [
+  { name: 'sc1', pageSize: 1024, reserve: 16, hmac: 0, kdf: 4000, algo: 'sha1' },
+  { name: 'sc2', pageSize: 1024, reserve: 48, hmac: 20, kdf: 4000, algo: 'sha1' },
+  { name: 'sc3', pageSize: 1024, reserve: 48, hmac: 20, kdf: 64000, algo: 'sha1' },
+  { name: 'sc4', pageSize: 4096, reserve: 80, hmac: 64, kdf: 256000, algo: 'sha512' },
+];
+
+function decPage(page, key, cfg, isP1) {
+  const reserveOff = cfg.pageSize - cfg.reserve;
+  const ct = page.subarray(isP1 ? 16 : 0, reserveOff);
+  const reserved = page.subarray(reserveOff, cfg.pageSize);
+  const iv = cfg.hmac ? reserved.subarray(cfg.hmac, cfg.hmac + 16) : reserved.subarray(0, 16);
+  if (iv.length < 16) return null;
+  try {
+    const d = crypto.createDecipheriv('aes-256-cbc', key, iv);
+    d.setAutoPadding(false);
+    return Buffer.concat([d.update(ct), d.final()]);
+  } catch {
+    return null;
+  }
+}
+const validP1 = (pt) => pt && pt.length > 8 && pt[5] === 64 && pt[6] === 32 && pt[7] === 32;
+
+/** key = MD5(IMEI+uin)[:7] over candidate imeis × uins, plus any saved/raw keys. */
+function computeKeyCandidates(imeis, uins, savedKeys) {
+  const set = new Set((savedKeys || []).filter(Boolean));
+  for (const im of imeis || []) for (const u of uins || []) set.add(md5(String(im) + String(u)).slice(0, 7));
+  return [...set];
+}
+
+/**
+ * Brute passphrases × configs (+ raw 32-byte keys), then decrypt all pages.
+ * @returns {{decrypted: Buffer, cfg: string, pass?: string}|null}
+ */
+function deriveAndDecrypt(raw, passphrases, rawKeys) {
+  if (!raw || raw.length < 1024) return null;
+  const salt = raw.subarray(0, 16);
+  let hit = null;
+  for (const cfg of CONFIGS) {
+    for (const p of passphrases || []) {
+      const k = crypto.pbkdf2Sync(Buffer.from(p, 'utf8'), salt, cfg.kdf, 32, cfg.algo === 'sha512' ? 'sha512' : 'sha1');
+      if (validP1(decPage(raw.subarray(0, cfg.pageSize), k, cfg, true))) { hit = { key: k, cfg, pass: p }; break; }
+    }
+    if (hit) break;
+    for (const rk of rawKeys || []) {
+      const kb = Buffer.isBuffer(rk) ? rk : Buffer.from(rk, 'hex');
+      if (kb.length === 32 && validP1(decPage(raw.subarray(0, cfg.pageSize), kb, cfg, true))) { hit = { key: kb, cfg }; break; }
+    }
+    if (hit) break;
+  }
+  if (!hit) return null;
+  const { key, cfg } = hit;
+  const n = Math.floor(raw.length / cfg.pageSize);
+  const out = [];
+  for (let i = 0; i < n; i++) {
+    const page = raw.subarray(i * cfg.pageSize, (i + 1) * cfg.pageSize);
+    const pt = decPage(page, key, cfg, i === 0);
+    const full = Buffer.alloc(cfg.pageSize);
+    if (i === 0) { Buffer.from('SQLite format 3\0').copy(full, 0); if (pt) pt.copy(full, 16); } else if (pt) pt.copy(full, 0);
+    out.push(full);
+  }
+  return { decrypted: Buffer.concat(out), cfg: cfg.name, pass: hit.pass };
+}
+
+/**
+ * Parse a DECRYPTED EnMicroMsg.db → vault events (wechat adapter shape).
+ * @param Database better-sqlite3 ctor (injected). @param self the user's wxid.
+ */
+function parseEvents(Database, dbPath, self) {
+  const src = new Database(dbPath, { readonly: true });
+  const events = [];
+  try {
+    const nameOf = new Map();
+    try {
+      for (const r of src.prepare('SELECT username,nickname,conRemark FROM rcontact').all()) {
+        nameOf.set(r.username, (r.conRemark && r.conRemark.trim()) || r.nickname || r.username);
+      }
+    } catch { /* contacts optional */ }
+    const rows = src.prepare(
+      'SELECT msgId,type,isSend,createTime,talker,content FROM message ' +
+      "WHERE type=1 ORDER BY createTime DESC LIMIT 5000",
+    ).all();
+    for (const r of rows) {
+      const isGroup = /@chatroom$/.test(r.talker || '');
+      let text = r.content || '';
+      let senderWxid = r.isSend ? self : r.talker;
+      if (isGroup && !r.isSend) {
+        const c = text.indexOf(':');
+        if (c > 0) { senderWxid = text.slice(0, c); text = text.slice(c + 1).replace(/^\n/, '').trim(); }
+      }
+      if (!text || !/[一-鿿A-Za-z0-9]/.test(text)) continue;
+      const occurredAt = Number(r.createTime) || 0; // already ms in WeChat
+      if (!occurredAt) continue;
+      const peer = String(r.talker || '');
+      const actor = `person-wechat-${senderWxid || self}`;
+      const participants = [actor];
+      participants.push(isGroup ? `group-wechat-${peer}` : `person-wechat-${peer}`);
+      events.push({
+        type: 'event', subtype: 'message', id: `wechat:${r.msgId}`,
+        occurredAt, actor, participants,
+        content: { text: isGroup ? `[群${nameOf.get(peer) || peer}] ${text}` : text },
+        topics: isGroup ? [`group-wechat-${peer}`] : undefined,
+        source: {
+          adapter: 'wechat', adapterVersion: '0.1.0', originalId: String(r.msgId),
+          capturedAt: occurredAt, capturedBy: 'sqlite',
+        },
+        ingestedAt: Date.now(),
+      });
+    }
+  } finally {
+    src.close();
+  }
+  return events;
+}
+
+module.exports = { computeKeyCandidates, deriveAndDecrypt, parseEvents };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chainlesschain/personal-data-hub",
-  "version": "0.4.30",
+  "version": "0.4.32",
   "description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
   "type": "commonjs",
   "main": "lib/index.js",
@@ -73,7 +73,10 @@
     "./adapters/messaging-whatsapp": "./lib/adapters/messaging-whatsapp/index.js",
     "./sidecar": "./lib/sidecar/index.js",
     "./forensics/leaf-salvage": "./lib/forensics/leaf-salvage.js",
-    "./forensics/salvage-ingest": "./lib/forensics/salvage-ingest.js"
+    "./forensics/salvage-ingest": "./lib/forensics/salvage-ingest.js",
+    "./forensics/qq-nt-collect": "./lib/forensics/qq-nt-collect.js",
+    "./forensics/wechat-collect": "./lib/forensics/wechat-collect.js",
+    "./forensics/plaintext-db-collect": "./lib/forensics/plaintext-db-collect.js"
   },
   "scripts": {
     "test": "vitest run",
@@ -104,7 +107,8 @@
   "optionalDependencies": {
     "imapflow": "^1.0.183",
     "adm-zip": "^0.5.16",
-    "iconv-lite": "^0.6.3"
+    "iconv-lite": "^0.6.3",
+    "pdf-parse": "^1.1.1"
   },
   "devDependencies": {
     "vitest": "^4.1.5"