@chainlesschain/personal-data-hub 0.3.8 → 0.4.0

package/lib/adapters/qq-pc/nt-db-reader.js

@@ -0,0 +1,210 @@
+"use strict";
+
+/**
+ * QQ NT **desktop (PC)** local-DB reader — 本地直读样板, ported from the
+ * Douyin / wechat-pc samples to QQ NT (新版 PC QQ, Electron 9.x).
+ *
+ * Source: `nt_msg.db` under the QQ data dir (e.g.
+ *   %APPDATA%\Tencent\QQ\nt_qq_<hash>\nt_db\nt_msg.db).
+ * Message tables: `c2c_msg_table` (1-on-1) + `group_msg_table` (群).
+ *
+ * ⚠️ HONEST CAVEAT (v0.1): unlike Douyin's plaintext im.db, QQ NT is the
+ * hardest of the three:
+ *   - SQLCipher-encrypted (decrypt to plaintext first — recommended).
+ *   - Column names are NUMERIC + obfuscated ("40050", "40011", ...) and
+ *     DRIFT across QQ versions.
+ *   - Message bodies are protobuf BLOBs, not plain text — full text needs
+ *     per-type protobuf decoding that must be tuned on a real DB.
+ *
+ * So this reader is deliberately DEFENSIVE + LOUD rather than clever:
+ *   - It probes a table's columns and resolves time/type/sender/peer/content
+ *     against candidate lists (readable names FIRST, then known numeric ids).
+ *   - It extracts text ONLY when the resolved content column is a real
+ *     string; otherwise text=null but the FULL raw row is preserved in
+ *     `rawRow` so nothing is lost and a later protobuf decoder can fill it.
+ *   - The diagnostic reports exactly which tables + columns were resolved,
+ *     so the user/UI sees what worked instead of silently getting 0 rows.
+ *
+ * Test seam: inject `_databaseClass` to bypass the native dual-load.
+ */
+
+function loadDatabaseClass() {
+  for (const mod of ["better-sqlite3-multiple-ciphers", "better-sqlite3"]) {
+    let cls;
+    try {
+      // eslint-disable-next-line global-require
+      cls = require(mod);
+    } catch (_e) {
+      continue;
+    }
+    try {
+      const probe = new cls(":memory:");
+      probe.close();
+      return cls;
+    } catch (_e) {
+      // ABI mismatch — try next
+    }
+  }
+  throw new Error(
+    "qq-pc-nt-db-reader: neither better-sqlite3-multiple-ciphers nor better-sqlite3 loaded — both ABI-mismatched",
+  );
+}
+
+function trySelect(db, sql) {
+  try {
+    return db.prepare(sql).all();
+  } catch (_e) {
+    return null;
+  }
+}
+
+function pickCol(columns, candidates) {
+  for (const c of candidates) {
+    if (columns.has(c)) return c;
+  }
+  return null;
+}
+
+function normalizeEpochMs(v) {
+  if (typeof v !== "number" || !Number.isFinite(v) || v <= 0) return null;
+  if (v > 1e15) return Math.floor(v / 1000); // µs
+  if (v > 1e12) return Math.floor(v); // ms
+  return Math.floor(v * 1000); // seconds
+}
+
+// Candidate column names per field — readable names first (decrypted /
+// re-exported DBs sometimes have them), then the known QQ NT numeric ids.
+// Tune the numeric lists on a real device; unknowns just fall through to
+// rawRow (loud diagnostic), never a silent 0.
+const COL_CANDIDATES = Object.freeze({
+  msgId: ["msgId", "msg_id", "40001", "40020"],
+  time: ["msgTime", "time", "timestamp", "40050"],
+  type: ["msgType", "type", "40011", "40012"],
+  sender: ["senderUin", "sender", "senderUid", "40033", "40030"],
+  peer: ["peerUin", "peer", "peerUid", "40021", "40027"],
+  content: ["content", "text", "msgContent", "40080", "40800"],
+});
+
+/**
+ * Open a QQ NT DB (plaintext OR SQLCipher-with-key). Mirrors wechat-pc.
+ */
+function openNtDb(dbPath, opts = {}) {
+  const Database = opts._databaseClass || loadDatabaseClass();
+  const key = typeof opts.key === "string" && opts.key.length > 0 ? opts.key : null;
+  if (!key) {
+    const db = new Database(dbPath, { readonly: true });
+    try {
+      db.prepare("SELECT count(*) AS n FROM sqlite_master").get();
+      return { db, mode: "plaintext" };
+    } catch (err) {
+      try { db.close(); } catch (_e) { /* ignore */ }
+      const e = new Error(
+        `qq-pc-nt-db-reader: db is not plaintext SQLite (decrypt nt_msg.db first, or pass --key): ${err.message}`,
+      );
+      e.code = "QQ_PC_NEEDS_KEY";
+      throw e;
+    }
+  }
+  const keyExpr = /^[0-9a-fA-F]{64}$/.test(key) ? `"x'${key}'"` : `'${key}'`;
+  const db = new Database(dbPath, { readonly: true });
+  try {
+    db.pragma(`key = ${keyExpr}`);
+    db.exec("PRAGMA cipher_compatibility = 4");
+    db.prepare("SELECT count(*) AS n FROM sqlite_master").get();
+    return { db, mode: "sqlcipher" };
+  } catch (err) {
+    try { db.close(); } catch (_e) { /* ignore */ }
+    const e = new Error(
+      `qq-pc-nt-db-reader: SQLCipher open failed (key wrong, or decrypt to plaintext first): ${err.message}`,
+    );
+    e.code = "QQ_PC_DECRYPT_FAILED";
+    throw e;
+  }
+}
+
+function readMsgTable(db, tableName, isGroup, limit, diag) {
+  const info = trySelect(db, `PRAGMA table_info(${tableName})`);
+  if (!Array.isArray(info) || info.length === 0) return [];
+  const cols = new Set(info.map((r) => r.name));
+  const resolved = {
+    msgId: pickCol(cols, COL_CANDIDATES.msgId),
+    time: pickCol(cols, COL_CANDIDATES.time),
+    type: pickCol(cols, COL_CANDIDATES.type),
+    sender: pickCol(cols, COL_CANDIDATES.sender),
+    peer: pickCol(cols, COL_CANDIDATES.peer),
+    content: pickCol(cols, COL_CANDIDATES.content),
+  };
+  diag.resolvedColumns[tableName] = resolved;
+  // Select ALL columns so the full raw row is preserved (protobuf bodies,
+  // unknown columns) — we map the resolved fields on top of it.
+  const orderBy = resolved.time ? ` ORDER BY "${resolved.time}" DESC` : "";
+  const rows =
+    trySelect(db, `SELECT * FROM ${tableName}${orderBy} LIMIT ${limit}`) || [];
+  return rows.map((row, idx) => {
+    const rawTime = resolved.time ? row[resolved.time] : null;
+    const contentVal = resolved.content ? row[resolved.content] : null;
+    return {
+      msgId:
+        (resolved.msgId && row[resolved.msgId] != null && String(row[resolved.msgId])) ||
+        `${tableName}-${idx}`,
+      isGroup,
+      createdTimeMs:
+        typeof rawTime === "number" ? normalizeEpochMs(rawTime) : null,
+      type:
+        resolved.type && typeof row[resolved.type] === "number"
+          ? row[resolved.type]
+          : null,
+      senderUin: resolved.sender && row[resolved.sender] != null ? String(row[resolved.sender]) : null,
+      peerUin: resolved.peer && row[resolved.peer] != null ? String(row[resolved.peer]) : null,
+      // Only treat content as text when it's a real string (not a BLOB).
+      text: typeof contentVal === "string" ? contentVal : null,
+      // Preserve the full raw row so a later protobuf decoder loses nothing.
+      rawRow: row,
+    };
+  });
+}
+
+/**
+ * Read messages out of a QQ NT nt_msg.db. Returns `{messages, diagnostic}`.
+ * Reads c2c_msg_table + group_msg_table (whichever exist).
+ */
+function readQqNt(dbPath, opts = {}) {
+  if (typeof dbPath !== "string" || dbPath.length === 0) {
+    throw new TypeError("readQqNt: dbPath must be a non-empty string");
+  }
+  const limit =
+    Number.isInteger(opts.limitMessages) && opts.limitMessages > 0
+      ? opts.limitMessages
+      : 20_000;
+  const { db, mode } = openNtDb(dbPath, opts);
+  const diagnostic = {
+    messageCount: 0,
+    hadC2cTable: false,
+    hadGroupTable: false,
+    textCount: 0,
+    resolvedColumns: {},
+    mode,
+  };
+  const messages = [];
+  try {
+    const c2c = readMsgTable(db, "c2c_msg_table", false, limit, diagnostic);
+    if (diagnostic.resolvedColumns.c2c_msg_table) diagnostic.hadC2cTable = true;
+    const group = readMsgTable(db, "group_msg_table", true, limit, diagnostic);
+    if (diagnostic.resolvedColumns.group_msg_table) diagnostic.hadGroupTable = true;
+    for (const m of [...c2c, ...group]) {
+      messages.push(m);
+      if (typeof m.text === "string" && m.text.length > 0) diagnostic.textCount += 1;
+    }
+    diagnostic.messageCount = messages.length;
+  } finally {
+    try { db.close(); } catch (_e) { /* ignore */ }
+  }
+  return { messages, diagnostic };
+}
+
+module.exports = {
+  readQqNt,
+  openNtDb,
+  COL_CANDIDATES,
+  _internals: { loadDatabaseClass, normalizeEpochMs, pickCol, readMsgTable },
+};

package/lib/adapters/social-douyin/index.js

@@ -185,7 +185,27 @@ class DouyinAdapter {
   }
 
   async *sync(opts = {}) {
+    // ── 本地直读样板 (local direct-read) ─────────────────────────────────
+    // The most reliable Douyin path: point at a locally-present
+    // `<uid>_im.db` (pulled from a rooted device or copied off the phone)
+    // and read 私信 + 联系人 straight out of the plaintext SQLite — no ADB
+    // orchestration, no snapshot-JSON round trip, no X-Bogus signing.
+    //
+    // Routing:
+    //   1. opts.imDbPath        — explicit IM-db path → direct read
+    //   2. opts.inputPath that sniffs as a SQLite file → direct read
+    //      (so `cc hub sync-adapter social-douyin --input <uid>_im.db`
+    //       just works); a non-SQLite inputPath is a snapshot JSON.
+    //   3. opts.dbPath (legacy) — Phase 13.3 video-history tables.
+    if (typeof opts.imDbPath === "string" && opts.imDbPath.length > 0) {
+      yield* this._syncViaImDb({ ...opts, dbPath: opts.imDbPath });
+      return;
+    }
     if (typeof opts.inputPath === "string" && opts.inputPath.length > 0) {
+      if (this._looksLikeSqlite(opts.inputPath)) {
+        yield* this._syncViaImDb({ ...opts, dbPath: opts.inputPath });
+        return;
+      }
       yield* this._syncViaSnapshot(opts);
       return;
     }
@@ -195,10 +215,107 @@ class DouyinAdapter {
       return;
     }
     throw new Error(
-      "social-douyin.sync: needs opts.inputPath (snapshot mode, Android in-APK cc) OR opts.dbPath (sqlite mode, legacy device-pull)",
+      "social-douyin.sync: needs opts.imDbPath / opts.inputPath (<uid>_im.db or snapshot JSON) OR opts.dbPath (legacy video-history sqlite)",
     );
   }
 
+  /**
+   * Cheap SQLite-file sniff via the 16-byte magic header
+   * ("SQLite format 3\0"). Lets sync() auto-route a `--input <uid>_im.db`
+   * to direct IM read vs treating a `.json` snapshot as SQLite. Returns
+   * false on any read error (caller falls back to snapshot path).
+   */
+  _looksLikeSqlite(filePath) {
+    try {
+      const fd = this._deps.fs.openSync(filePath, "r");
+      try {
+        const buf = Buffer.alloc(16);
+        this._deps.fs.readSync(fd, buf, 0, 16, 0);
+        return buf.toString("latin1").startsWith("SQLite format 3");
+      } finally {
+        this._deps.fs.closeSync(fd);
+      }
+    } catch (_e) {
+      return false;
+    }
+  }
+
+  /**
+   * 本地直读 <uid>_im.db — open the plaintext SQLite directly and yield
+   * message + contact raw events. Reuses social-douyin-adb/im-db-parser so
+   * the defensive column-picker / epoch-normalize / content-JSON logic stays
+   * byte-identical with the ADB-pull path (single source of truth). Emits
+   * the SAME composite originalIds as the snapshot path so re-syncing the
+   * same db (via either route) is idempotent.
+   */
+  async *_syncViaImDb(opts) {
+    const dbPath = opts.dbPath;
+    if (!dbPath || !this._deps.fs.existsSync(dbPath)) return;
+    // eslint-disable-next-line global-require
+    const { parseImDb } = require("../social-douyin-adb/im-db-parser");
+    const parseOpts = {};
+    if (Number.isInteger(opts.limitMessages)) parseOpts.limitMessages = opts.limitMessages;
+    if (Number.isInteger(opts.limitContacts)) parseOpts.limitContacts = opts.limitContacts;
+    if (this._deps.dbDriverFactory) parseOpts._databaseClass = this._deps.dbDriverFactory();
+
+    const { messages, contacts, diagnostic } = parseImDb(dbPath, parseOpts);
+    if (typeof opts.onProgress === "function") {
+      try {
+        opts.onProgress({ phase: "im-db-parsed", adapter: NAME, ...diagnostic });
+      } catch (_e) { /* progress is best-effort */ }
+    }
+
+    const include = opts.include || {};
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+    const fallbackCapturedAt = Date.now();
+    let emitted = 0;
+
+    if (include[KIND_MESSAGE] !== false) {
+      for (const m of messages) {
+        if (emitted >= limit) return;
+        if (!m || typeof m !== "object") continue;
+        // Mirror snapshot-builder's composite-id strategy verbatim so the
+        // direct-read and snapshot routes converge on one originalId.
+        const idPart =
+          m.conversationId && m.createdTimeMs
+            ? `${m.conversationId}-${m.createdTimeMs}`
+            : m.senderUid && m.createdTimeMs
+              ? `${m.senderUid}-${m.createdTimeMs}`
+              : `msg-${emitted}`;
+        const capturedAt =
+          typeof m.createdTimeMs === "number" && m.createdTimeMs > 0
+            ? m.createdTimeMs
+            : fallbackCapturedAt;
+        yield {
+          adapter: NAME,
+          kind: KIND_MESSAGE,
+          originalId: stableOriginalId(KIND_MESSAGE, `msg-${idPart}`),
+          capturedAt,
+          payload: { kind: KIND_MESSAGE, ...m },
+        };
+        emitted += 1;
+      }
+    }
+
+    if (include[KIND_CONTACT] !== false) {
+      for (const c of contacts) {
+        if (emitted >= limit) return;
+        if (!c || typeof c !== "object") continue;
+        yield {
+          adapter: NAME,
+          kind: KIND_CONTACT,
+          originalId: stableOriginalId(
+            KIND_CONTACT,
+            c.uid ? `contact-${c.uid}` : `contact-${emitted}`,
+          ),
+          capturedAt: fallbackCapturedAt,
+          payload: { kind: KIND_CONTACT, ...c },
+        };
+        emitted += 1;
+      }
+    }
+  }
+
   async *_syncViaSnapshot(opts) {
     const raw = this._deps.fs.readFileSync(opts.inputPath, "utf-8");
     const snapshot = JSON.parse(raw);
@@ -331,6 +448,12 @@ class DouyinAdapter {
     if (kind === KIND_SEARCH) {
       return normalizeSearch(p, raw, ingestedAt);
     }
+    if (kind === KIND_MESSAGE) {
+      return normalizeMessage(p, raw, ingestedAt);
+    }
+    if (kind === KIND_CONTACT) {
+      return normalizeContact(p, raw, ingestedAt);
+    }
     throw new Error(`DouyinAdapter.normalize: unknown kind ${kind}`);
   }
 }
@@ -502,6 +625,76 @@ function normalizeSearch(p, raw, ingestedAt) {
   };
 }
 
+function normalizeMessage(p, raw, ingestedAt) {
+  // IM private message from <uid>_im.db (snapshot or 本地直读). Becomes one
+  // MESSAGE event. We don't reliably know the self uid here, so actor stays
+  // person-self and the real sender is preserved in extra.senderUid (the
+  // consumer / EntityResolver can correlate it to a contact person).
+  const occurredAt =
+    parseTime(p.createdTimeMs) || raw.capturedAt || ingestedAt;
+  const source = buildSource(raw, occurredAt, CAPTURED_BY.SQLITE);
+  const text = typeof p.text === "string" ? p.text : "";
+  return {
+    events: [{
+      id: newId(),
+      type: ENTITY_TYPES.EVENT,
+      subtype: EVENT_SUBTYPES.MESSAGE,
+      occurredAt,
+      actor: "person-self",
+      content: {
+        title: text ? text.slice(0, 80) : "(非文本消息)",
+        text,
+      },
+      ingestedAt,
+      source,
+      extra: {
+        platform: "douyin",
+        channel: "im",
+        senderUid: p.senderUid || null,
+        conversationId: p.conversationId || null,
+        readStatus: typeof p.readStatus === "number" ? p.readStatus : null,
+        // Preserve the raw content blob for non-text message types (stickers
+        // / voice / video) so a richer consumer can decode them later.
+        contentBlob: typeof p.contentBlob === "string" ? p.contentBlob : null,
+      },
+    }],
+    persons: [], places: [], items: [], topics: [],
+  };
+}
+
+function normalizeContact(p, raw, ingestedAt) {
+  // SIMPLE_USER row from <uid>_im.db → a contact Person. SIMPLE_USER has no
+  // per-row timestamp, so occurredAt falls back to capturedAt.
+  const uid =
+    (typeof p.uid === "string" && p.uid) ||
+    (typeof p.uid === "number" && String(p.uid)) ||
+    null;
+  const occurredAt = raw.capturedAt || ingestedAt;
+  const source = buildSource(raw, occurredAt, CAPTURED_BY.SQLITE);
+  const identifiers = {};
+  if (uid) identifiers["douyin-uid"] = [uid];
+  if (p.shortId) identifiers["douyin-short-id"] = [String(p.shortId)];
+  return {
+    events: [],
+    persons: [{
+      id: uid ? `person-douyin-${uid}` : `person-douyin-${newId()}`,
+      type: ENTITY_TYPES.PERSON,
+      subtype: PERSON_SUBTYPES.CONTACT,
+      names: [p.name || "(unnamed)"],
+      ingestedAt,
+      source,
+      identifiers,
+      extra: {
+        platform: "douyin",
+        avatarUrl: p.avatarUrl || null,
+        // 0/1/2 = none / following / mutual (Douyin follow_status)
+        followStatus: typeof p.followStatus === "number" ? p.followStatus : null,
+      },
+    }],
+    places: [], items: [], topics: [],
+  };
+}
+
 module.exports = {
   DouyinAdapter,
   NAME,

package/lib/adapters/wechat/wechat-adapter.js

@@ -72,7 +72,7 @@ class WechatAdapter {
     };
   }
 
-  async authenticate() {
+  async authenticate(ctx = {}) {
     // No server auth; sanity check the on-disk state.
     if (!this._dbPath || !fs.existsSync(this._dbPath)) {
       return { ok: false, reason: "DB_NOT_PULLED", error: `DB path missing: ${this._dbPath}` };
@@ -80,6 +80,12 @@ class WechatAdapter {
     if (!this._keyProvider || typeof this._keyProvider.getKey !== "function") {
       return { ok: false, reason: "NO_KEY_PROVIDER", error: "keyProvider required" };
     }
+    // Readiness probe — DB + key provider present means "configured". Do NOT
+    // invoke the (possibly frida-backed, expensive/side-effectful) key
+    // provider during a readiness check; the real sync exercises it.
+    if (ctx && ctx.readinessOnly) {
+      return { ok: true, mode: "configured" };
+    }
     try {
       const key = await this._keyProvider.getKey();
       if (!key) return { ok: false, reason: "EMPTY_KEY", error: "keyProvider returned empty key" };