@chainlesschain/personal-data-hub 0.4.25 → 0.4.27

package/lib/forensics/leaf-salvage.js

@@ -0,0 +1,185 @@
+"use strict";
+/*
+ * PDH SQLite leaf-page record salvager — library form.
+ *
+ * Method B (`/proc/<pid>/mem` 内存扫描) dumps decrypted pages, but for DBs whose
+ * page cache is SCATTERED (not contiguous mmap) the rebuilt file is "malformed"
+ * (valid header, broken b-tree). This salvages the DATA anyway: it scans a dump
+ * (or concatenated dumps) for SQLite **table b-tree leaf pages** (type 0x0D) and
+ * parses each page's records directly — order-independent, exactly what sqlite3
+ * `.recover` does, but standalone (platform-tools sqlite3 lacks .recover).
+ *
+ * This file is the bundle-able home for the parser; the standalone CLI tool
+ * `scripts/android/pdh-sqlite-leaf-salvage.js` re-exports from here. Lives in
+ * pdh lib so the Android cc bundle (and `cc hub salvage`) can call it on-device.
+ *
+ * Output: array of {rowid, cols:[...]} — raw positional column tuples (leaf
+ * pages carry no column names). Map to a schema downstream (salvage-mapper.js).
+ *
+ * Authorization: only on data you are entitled to (your own device/account).
+ * Docs: docs/internal/pdh-db-decryption-runbook.md (Method B + reconstruction).
+ */
+const fs = require("node:fs");
+
+function readVarint(buf, off) {
+  // SQLite varint: up to 9 bytes, big-endian, high bit = continuation.
+  let result = 0n;
+  let i = 0;
+  for (; i < 8; i++) {
+    const b = buf[off + i];
+    if (b === undefined) return [null, off + i];
+    result = (result << 7n) | BigInt(b & 0x7f);
+    if ((b & 0x80) === 0) return [result, off + i + 1];
+  }
+  // 9th byte uses all 8 bits
+  const b9 = buf[off + 8];
+  if (b9 === undefined) return [null, off + 9];
+  result = (result << 8n) | BigInt(b9);
+  return [result, off + 9];
+}
+
+function serialTypeSize(t) {
+  // t is a BigInt
+  const n = Number(t);
+  if (n === 0 || n === 8 || n === 9 || n === 12 || n === 13) {
+    return n >= 12 ? (n % 2 === 0 ? (n - 12) / 2 : (n - 13) / 2) : 0;
+  }
+  if (n === 1) return 1;
+  if (n === 2) return 2;
+  if (n === 3) return 3;
+  if (n === 4) return 4;
+  if (n === 5) return 6;
+  if (n === 6) return 8;
+  if (n === 7) return 8;
+  if (n >= 12) return n % 2 === 0 ? (n - 12) / 2 : (n - 13) / 2;
+  return 0;
+}
+
+function readValue(buf, off, t) {
+  const n = Number(t);
+  const sz = serialTypeSize(t);
+  if (n === 0) return [null, off];
+  if (n === 8) return [0, off];
+  if (n === 9) return [1, off];
+  if (n >= 1 && n <= 6) {
+    let v = 0n;
+    for (let i = 0; i < sz; i++) v = (v << 8n) | BigInt(buf[off + i] || 0);
+    // sign-extend
+    const bits = BigInt(sz * 8);
+    if (v >= 1n << (bits - 1n)) v -= 1n << bits;
+    const num = Number(v);
+    return [Number.isSafeInteger(num) ? num : v.toString(), off + sz];
+  }
+  if (n === 7) return [buf.readDoubleBE(off), off + 8];
+  if (n >= 13 && n % 2 === 1) {
+    // text
+    return [buf.toString("utf8", off, off + sz), off + sz];
+  }
+  // blob (n>=12 even) — return length marker, not raw bytes
+  return [`<blob:${sz}>`, off + sz];
+}
+
+// Parse one table-leaf page at `base`. Returns array of {rowid, cols} or null.
+function parseLeafPage(buf, base, pageSize, minCols) {
+  if (buf[base] !== 0x0d) return null; // 0x0D = table b-tree leaf
+  if (base + pageSize > buf.length) return null;
+  const numCells = (buf[base + 3] << 8) | buf[base + 4];
+  if (numCells <= 0 || numCells > Math.floor(pageSize / 4)) return null;
+  // cell-content-start (bytes 5-6; 0 means 65536) — must sit after the cell
+  // pointer array and within the page. This guard rejects the false positives a
+  // finer (unaligned) stride would otherwise hit on random 0x0D bytes.
+  let cellStart = (buf[base + 5] << 8) | buf[base + 6];
+  if (cellStart === 0) cellStart = 65536;
+  const hdrEndMin = 8 + numCells * 2;
+  if (cellStart < hdrEndMin || cellStart > pageSize) return null;
+  const out = [];
+  const ptrBase = base + 8;
+  for (let c = 0; c < numCells; c++) {
+    const ptr = (buf[ptrBase + c * 2] << 8) | buf[ptrBase + c * 2 + 1];
+    if (ptr < 8 || ptr >= pageSize) continue;
+    let off = base + ptr;
+    try {
+      const [payloadLen, o1] = readVarint(buf, off); off = o1;
+      if (payloadLen === null || payloadLen <= 0n || payloadLen > BigInt(pageSize)) continue;
+      const [rowid, o2] = readVarint(buf, off); off = o2;
+      // record header
+      const recStart = off;
+      const [hdrLen, o3] = readVarint(buf, off); off = o3;
+      if (hdrLen === null || hdrLen <= 0n) continue;
+      const hdrEnd = recStart + Number(hdrLen);
+      const serials = [];
+      while (off < hdrEnd) {
+        const [st, oN] = readVarint(buf, off); off = oN;
+        if (st === null) break;
+        serials.push(st);
+      }
+      let vOff = hdrEnd;
+      const cols = [];
+      for (const st of serials) {
+        const [val, vN] = readValue(buf, vOff, st);
+        cols.push(val); vOff = vN;
+      }
+      if (cols.length >= minCols) out.push({ rowid: rowid === null ? null : rowid.toString(), cols });
+    } catch (_e) { /* skip malformed cell */ }
+  }
+  return out.length ? out : null;
+}
+
+/**
+ * Scan an in-memory dump buffer for table-leaf pages and return all salvaged
+ * records (deduped). Engine-agnostic — works on any decrypted-page dump.
+ *
+ * @param {Buffer} buf
+ * @param {{pageSize?: number, minCols?: number, unaligned?: boolean, stride?: number}} [opts]
+ *   - pageSize: SQLite page size (default 4096)
+ *   - minCols:  drop records with fewer columns (default 3)
+ *   - unaligned: scan at a finer stride (512) to catch pages not 4096-aligned
+ *     in a malloc'd page cache; ~8x slower, recovers pages the aligned scan
+ *     misses. The strengthened header validation rejects the extra false
+ *     positives.
+ *   - stride: explicit scan stride (overrides the unaligned default)
+ * @returns {{records: Array<{rowid: string|null, cols: any[]}>, pages: number}}
+ */
+function salvageBuffer(buf, opts = {}) {
+  if (!Buffer.isBuffer(buf)) {
+    throw new TypeError("salvageBuffer: buf must be a Buffer");
+  }
+  const pageSize = Number.isFinite(opts.pageSize) && opts.pageSize > 0 ? opts.pageSize : 4096;
+  const minCols = Number.isFinite(opts.minCols) && opts.minCols >= 0 ? opts.minCols : 3;
+  const unaligned = !!opts.unaligned;
+  const stride = Number.isFinite(opts.stride) && opts.stride > 0
+    ? opts.stride
+    : (unaligned ? 512 : pageSize);
+
+  let pages = 0;
+  const records = [];
+  const seen = new Set(); // dedup overlapping finds
+  for (let base = 0; base + 8 <= buf.length; base += stride) {
+    if (buf[base] !== 0x0d) continue; // cheap pre-filter before full parse
+    const recs = parseLeafPage(buf, base, pageSize, minCols);
+    if (!recs) continue;
+    pages++;
+    for (const r of recs) {
+      const key = r.rowid + " " + JSON.stringify(r.cols);
+      if (seen.has(key)) continue;
+      seen.add(key);
+      records.push(r);
+    }
+  }
+  return { records, pages };
+}
+
+/** Read a dump file and salvage records from it. See {@link salvageBuffer}. */
+function salvageFile(filePath, opts = {}) {
+  const buf = fs.readFileSync(filePath);
+  return salvageBuffer(buf, opts);
+}
+
+module.exports = {
+  readVarint,
+  serialTypeSize,
+  readValue,
+  parseLeafPage,
+  salvageBuffer,
+  salvageFile,
+};

package/lib/forensics/salvage-ingest.js

@@ -0,0 +1,160 @@
+"use strict";
+/*
+ * Generic salvage → vault ingest (multi-app, correct source attribution).
+ *
+ * The leaf-page salvager recovers raw {rowid, cols} tuples from a /proc/mem
+ * dump of ANY app. This maps them into MESSAGE events and writes them straight
+ * to the vault with the CORRECT per-app `source.adapter` — so 抖音 data shows as
+ * social-douyin, 头条 as social-toutiao, 微信 as wechat, etc., instead of all
+ * being mis-attributed to one adapter.
+ *
+ * Why direct vault.putBatch (not registry.syncAdapter): the registry path runs
+ * an adapter's normalize(), which hard-stamps that adapter's name as the source.
+ * For salvaged cross-app data we need to stamp the source per the app the dump
+ * came from. Building events here + putBatch keeps attribution correct and is
+ * engine-agnostic. Events are tagged `extra.capturedBy = "mem-salvage"` for
+ * provenance + `extra.salvaged = true`.
+ *
+ * Authorization: only on data you are entitled to (your own device/account).
+ * Docs: docs/internal/pdh-db-decryption-runbook.md (Method B).
+ */
+const { newId } = require("../ids");
+const { salvageFile } = require("./leaf-salvage");
+const {
+  mapMsgRecords,
+  inferMsgColumns,
+} = require("../adapters/social-douyin-adb/salvage-mapper");
+
+const SALVAGE_VERSION = "salvage-0.1";
+
+// appKey → { sourceAdapter (vault source.adapter, = the app's canonical adapter
+// name so byApp aggregation attributes correctly), platform }. Unknown apps
+// fall back to a "salvage:<app>" source so they're still distinct + traceable.
+const APP_SALVAGE = Object.freeze({
+  douyin: { sourceAdapter: "social-douyin", platform: "douyin" },
+  toutiao: { sourceAdapter: "social-toutiao", platform: "toutiao" },
+  kuaishou: { sourceAdapter: "social-kuaishou", platform: "kuaishou" },
+  xiaohongshu: { sourceAdapter: "social-xiaohongshu", platform: "xiaohongshu" },
+  weibo: { sourceAdapter: "social-weibo", platform: "weibo" },
+  wechat: { sourceAdapter: "wechat", platform: "wechat" },
+  qq: { sourceAdapter: "qq", platform: "qq" },
+});
+
+function resolveApp(app) {
+  const key = String(app || "douyin").toLowerCase();
+  return APP_SALVAGE[key] || { sourceAdapter: `salvage:${key}`, platform: key };
+}
+
+// Stable, content-derived hash → dedup key. Same message salvaged again
+// (re-scan) collides on ON CONFLICT(source_adapter, source_original_id) and
+// updates rather than duplicating. Tiny FNV-1a over the salient fields.
+function _stableKey(parts) {
+  const s = parts.join("|");
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0;
+  }
+  return h.toString(16).padStart(8, "0");
+}
+
+/**
+ * Map salvaged records → MESSAGE events tagged with the per-app source.
+ * @param {Array<{rowid,cols}>} records
+ * @param {{ app?: string, columns?: string[], now?: number }} [opts]
+ * @returns {{ events: object[], mapped: number, columns: string[], sourceAdapter: string }}
+ */
+function buildSalvageEvents(records, opts = {}) {
+  const cfg = resolveApp(opts.app);
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const columns = Array.isArray(opts.columns) && opts.columns.length
+    ? opts.columns
+    : inferMsgColumns(records || []);
+  const messages = mapMsgRecords(records || [], columns);
+  const events = [];
+  for (const m of messages) {
+    const text = typeof m.text === "string" ? m.text : "";
+    const occurredAt = Number.isFinite(m.createdTimeMs) && m.createdTimeMs > 0
+      ? m.createdTimeMs
+      : now;
+    const key = _stableKey([
+      cfg.platform,
+      String(m.conversationId || m.senderUid || ""),
+      String(occurredAt),
+      text.slice(0, 64),
+    ]);
+    events.push({
+      id: newId(),
+      type: "event",
+      subtype: "message",
+      occurredAt,
+      actor: "person-self",
+      content: {
+        title: text ? text.slice(0, 80) : "(非文本消息)",
+        text,
+      },
+      ingestedAt: now,
+      source: {
+        adapter: cfg.sourceAdapter,
+        adapterVersion: SALVAGE_VERSION,
+        originalId: `salvage:${cfg.platform}:${key}`,
+        capturedAt: occurredAt,
+        // schema enum: export|api|sqlite|accessibility|ocr|manual. Salvaged
+        // pages ARE SQLite rows; mem-salvage provenance lives in extra.salvaged.
+        capturedBy: "sqlite",
+      },
+      extra: {
+        platform: cfg.platform,
+        channel: "im",
+        salvaged: true,
+        senderUid: m.senderUid || null,
+        conversationId: m.conversationId || null,
+        contentBlob: typeof m.contentBlob === "string" ? m.contentBlob : null,
+      },
+    });
+  }
+  return { events, mapped: messages.length, columns, sourceAdapter: cfg.sourceAdapter };
+}
+
+/**
+ * Salvage a memory dump and write the recovered messages straight into the
+ * vault with the correct per-app source. Returns counts.
+ *
+ * @param {object} vault  LocalVault (must expose putBatch)
+ * @param {string} dumpPath
+ * @param {{ app?: string, columns?: string[], unaligned?: boolean, pageSize?: number, minCols?: number, now?: number }} [opts]
+ * @returns {{ ingested: number, salvaged: number, app: string, sourceAdapter: string, leafPages: number }}
+ */
+function salvageDumpToVault(vault, dumpPath, opts = {}) {
+  if (!vault || typeof vault.putBatch !== "function") {
+    throw new TypeError("salvageDumpToVault: vault with putBatch required");
+  }
+  if (typeof dumpPath !== "string" || !dumpPath) {
+    throw new TypeError("salvageDumpToVault: dumpPath required");
+  }
+  const { records, pages } = salvageFile(dumpPath, {
+    pageSize: opts.pageSize,
+    minCols: opts.minCols,
+    unaligned: opts.unaligned,
+    stride: opts.stride,
+  });
+  const built = buildSalvageEvents(records, opts);
+  const res = built.events.length
+    ? vault.putBatch({ events: built.events })
+    : { events: 0 };
+  return {
+    ingested: res.events || 0,
+    salvaged: records.length,
+    mapped: built.mapped,
+    app: String(opts.app || "douyin").toLowerCase(),
+    sourceAdapter: built.sourceAdapter,
+    leafPages: pages,
+  };
+}
+
+module.exports = {
+  APP_SALVAGE,
+  resolveApp,
+  buildSalvageEvents,
+  salvageDumpToVault,
+};

package/lib/prompt-builder.js

@@ -40,6 +40,7 @@ const FACT_BLOCK_FOOTER = "END FACTS.";
 const NO_FACTS_HINT = "(FACTS is empty — the vault has nothing matching this question. Say so honestly.)";
 const TOTALS_HEADER = "TOTALS (authoritative entity counts from vault — use these for count questions, NOT FACTS length):";
 const AMOUNT_SUM_HEADER = "AMOUNT_SUM (authoritative SQL total of amount-bearing events — use for spending questions, NOT FACTS sums):";
+const CROSS_APP_HEADER = "CROSS_APP_OVERVIEW (跨 app 汇聚画像 — 各 app 活跃度/类型/消费/高频联系人，回答跨 app 与决策类问题时优先参考；为汇总信号，非逐条事实):";
 
 // ─── Fact summarization ─────────────────────────────────────────────────
 
@@ -134,6 +135,10 @@ function buildPrompt(opts) {
     opts.vaultTotals && typeof opts.vaultTotals === "object" ? opts.vaultTotals : null;
   const amountSummary =
     opts.amountSummary && typeof opts.amountSummary === "object" ? opts.amountSummary : null;
+  const crossAppOverview =
+    typeof opts.crossAppOverview === "string" && opts.crossAppOverview.length > 0
+      ? opts.crossAppOverview
+      : null;
 
   const trimmed = facts.slice(0, maxFacts);
   const summaries = trimmed
@@ -170,6 +175,10 @@ function buildPrompt(opts) {
   if (amountSummary && Number.isFinite(amountSummary.total) && amountSummary.count > 0) {
     userContent += `\n${AMOUNT_SUM_HEADER}\n${JSON.stringify(amountSummary, null, 2)}\n`;
   }
+  // CROSS_APP_OVERVIEW — 跨 app 汇聚画像，置于 FACTS 前（同 TOTALS）。
+  if (crossAppOverview) {
+    userContent += `\n${CROSS_APP_HEADER}\n${crossAppOverview}\n`;
+  }
   userContent += `\n${FACT_BLOCK_HEADER}\n${factBody}\n${FACT_BLOCK_FOOTER}${truncatedNote}\n\nUSER QUESTION: ${question}`;
 
   return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chainlesschain/personal-data-hub",
-  "version": "0.4.25",
+  "version": "0.4.27",
   "description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
   "type": "commonjs",
   "main": "lib/index.js",
@@ -68,7 +68,9 @@
     "./adapters/messaging-qq": "./lib/adapters/messaging-qq/index.js",
     "./adapters/messaging-telegram": "./lib/adapters/messaging-telegram/index.js",
     "./adapters/messaging-whatsapp": "./lib/adapters/messaging-whatsapp/index.js",
-    "./sidecar": "./lib/sidecar/index.js"
+    "./sidecar": "./lib/sidecar/index.js",
+    "./forensics/leaf-salvage": "./lib/forensics/leaf-salvage.js",
+    "./forensics/salvage-ingest": "./lib/forensics/salvage-ingest.js"
   },
   "scripts": {
     "test": "vitest run",