@chainlesschain/personal-data-hub 0.4.26 → 0.4.27

package/__tests__/salvage-ingest.test.js

@@ -0,0 +1,97 @@
+"use strict";
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const { LocalVault } = require("../lib/vault");
+const { generateKeyHex } = require("../lib/key-providers");
+const {
+  buildSalvageEvents,
+  salvageDumpToVault,
+  resolveApp,
+} = require("../lib/forensics/salvage-ingest");
+
+// Build a real SQLite DB and treat its bytes as a memory dump; verify the
+// generic salvage→vault path recovers messages AND tags them with the correct
+// per-app source.adapter (multi-app de-silo). Real LocalVault → proves the
+// hand-built events pass schema validation + are searchable.
+describe("salvage-ingest — generic multi-app salvage → vault", () => {
+  let dir, dumpPath, vault, vdir;
+  const COLUMNS = ["msg_uuid", "conversation_id", "sender", "content", "created_time"];
+
+  beforeAll(() => {
+    const Database = require("better-sqlite3-multiple-ciphers");
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "salvage-ing-"));
+    dumpPath = path.join(dir, "u.db");
+    const db = new Database(dumpPath);
+    db.exec("CREATE TABLE msg(msg_uuid TEXT, conversation_id TEXT, sender INTEGER, content TEXT, created_time INTEGER)");
+    const ins = db.prepare("INSERT INTO msg VALUES(?,?,?,?,?)");
+    ins.run("u1", "conv-1", 111, "今天的会议改到下午三点 hi", 1700000000000);
+    ins.run("u2", "conv-1", 222, "收到 👌", 1700000001000);
+    db.close();
+
+    vdir = fs.mkdtempSync(path.join(os.tmpdir(), "salvage-vault-"));
+    vault = new LocalVault({ path: path.join(vdir, "v.db"), key: generateKeyHex() });
+    vault.open();
+  });
+
+  afterAll(() => {
+    try { vault.close(); } catch (_e) {}
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) {}
+    try { fs.rmSync(vdir, { recursive: true, force: true }); } catch (_e) {}
+  });
+
+  it("resolveApp maps known apps to canonical source adapters; unknown → salvage:<app>", () => {
+    expect(resolveApp("douyin").sourceAdapter).toBe("social-douyin");
+    expect(resolveApp("toutiao").sourceAdapter).toBe("social-toutiao");
+    expect(resolveApp("wechat").sourceAdapter).toBe("wechat");
+    expect(resolveApp("kuaishou").sourceAdapter).toBe("social-kuaishou");
+    expect(resolveApp("bogusapp").sourceAdapter).toBe("salvage:bogusapp");
+  });
+
+  it("buildSalvageEvents tags per-app source + builds valid message events", () => {
+    const recs = [{ rowid: "1", cols: ["u1", "conv-1", 111, "hello 世界", 1700000000000] }];
+    const built = buildSalvageEvents(recs, { app: "toutiao", columns: COLUMNS, now: 1700000099000 });
+    expect(built.events.length).toBe(1);
+    const e = built.events[0];
+    expect(e.source.adapter).toBe("social-toutiao");
+    expect(e.source.capturedBy).toBe("sqlite"); // schema enum; provenance in extra.salvaged
+    expect(e.subtype).toBe("message");
+    expect(e.content.text).toBe("hello 世界");
+    expect(e.extra.platform).toBe("toutiao");
+    expect(e.extra.salvaged).toBe(true);
+  });
+
+  it("salvageDumpToVault ingests with douyin source + events are searchable", () => {
+    const r = salvageDumpToVault(vault, dumpPath, { app: "douyin", columns: COLUMNS, now: 1700000099000 });
+    expect(r.app).toBe("douyin");
+    expect(r.sourceAdapter).toBe("social-douyin");
+    expect(r.ingested).toBe(2);
+    // events landed under the correct source + are searchable
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    const douyin = events.filter((e) => e.source && e.source.adapter === "social-douyin");
+    expect(douyin.length).toBe(2);
+    const texts = douyin.map((e) => e.content && e.content.text).sort();
+    expect(texts).toContain("收到 👌"); // UTF-8 emoji survives
+  });
+
+  it("same dump under a different app tags a different source (no cross-attribution)", () => {
+    const r = salvageDumpToVault(vault, dumpPath, { app: "toutiao", columns: COLUMNS, now: 1700000099000 });
+    expect(r.sourceAdapter).toBe("social-toutiao");
+    expect(r.ingested).toBe(2);
+    const events = vault.queryEvents({ limit: 100 }) || [];
+    expect(events.filter((e) => e.source && e.source.adapter === "social-toutiao").length).toBe(2);
+    // douyin events from prior test remain distinct
+    expect(events.filter((e) => e.source && e.source.adapter === "social-douyin").length).toBe(2);
+  });
+
+  it("re-ingesting the same dump dedups (stable originalId)", () => {
+    const before = (vault.queryEvents({ limit: 200 }) || []).length;
+    salvageDumpToVault(vault, dumpPath, { app: "douyin", columns: COLUMNS, now: 1700000099000 });
+    const after = (vault.queryEvents({ limit: 200 }) || []).length;
+    expect(after).toBe(before); // ON CONFLICT(source_adapter, source_original_id) updates, no dupes
+  });
+});

package/lib/forensics/salvage-ingest.js

@@ -0,0 +1,160 @@
+"use strict";
+/*
+ * Generic salvage → vault ingest (multi-app, correct source attribution).
+ *
+ * The leaf-page salvager recovers raw {rowid, cols} tuples from a /proc/mem
+ * dump of ANY app. This maps them into MESSAGE events and writes them straight
+ * to the vault with the CORRECT per-app `source.adapter` — so 抖音 data shows as
+ * social-douyin, 头条 as social-toutiao, 微信 as wechat, etc., instead of all
+ * being mis-attributed to one adapter.
+ *
+ * Why direct vault.putBatch (not registry.syncAdapter): the registry path runs
+ * an adapter's normalize(), which hard-stamps that adapter's name as the source.
+ * For salvaged cross-app data we need to stamp the source per the app the dump
+ * came from. Building events here + putBatch keeps attribution correct and is
+ * engine-agnostic. Events are tagged `extra.capturedBy = "mem-salvage"` for
+ * provenance + `extra.salvaged = true`.
+ *
+ * Authorization: only on data you are entitled to (your own device/account).
+ * Docs: docs/internal/pdh-db-decryption-runbook.md (Method B).
+ */
+const { newId } = require("../ids");
+const { salvageFile } = require("./leaf-salvage");
+const {
+  mapMsgRecords,
+  inferMsgColumns,
+} = require("../adapters/social-douyin-adb/salvage-mapper");
+
+const SALVAGE_VERSION = "salvage-0.1";
+
+// appKey → { sourceAdapter (vault source.adapter, = the app's canonical adapter
+// name so byApp aggregation attributes correctly), platform }. Unknown apps
+// fall back to a "salvage:<app>" source so they're still distinct + traceable.
+const APP_SALVAGE = Object.freeze({
+  douyin: { sourceAdapter: "social-douyin", platform: "douyin" },
+  toutiao: { sourceAdapter: "social-toutiao", platform: "toutiao" },
+  kuaishou: { sourceAdapter: "social-kuaishou", platform: "kuaishou" },
+  xiaohongshu: { sourceAdapter: "social-xiaohongshu", platform: "xiaohongshu" },
+  weibo: { sourceAdapter: "social-weibo", platform: "weibo" },
+  wechat: { sourceAdapter: "wechat", platform: "wechat" },
+  qq: { sourceAdapter: "qq", platform: "qq" },
+});
+
+function resolveApp(app) {
+  const key = String(app || "douyin").toLowerCase();
+  return APP_SALVAGE[key] || { sourceAdapter: `salvage:${key}`, platform: key };
+}
+
+// Stable, content-derived hash → dedup key. Same message salvaged again
+// (re-scan) collides on ON CONFLICT(source_adapter, source_original_id) and
+// updates rather than duplicating. Tiny FNV-1a over the salient fields.
+function _stableKey(parts) {
+  const s = parts.join("|");
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0;
+  }
+  return h.toString(16).padStart(8, "0");
+}
+
+/**
+ * Map salvaged records → MESSAGE events tagged with the per-app source.
+ * @param {Array<{rowid,cols}>} records
+ * @param {{ app?: string, columns?: string[], now?: number }} [opts]
+ * @returns {{ events: object[], mapped: number, columns: string[], sourceAdapter: string }}
+ */
+function buildSalvageEvents(records, opts = {}) {
+  const cfg = resolveApp(opts.app);
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const columns = Array.isArray(opts.columns) && opts.columns.length
+    ? opts.columns
+    : inferMsgColumns(records || []);
+  const messages = mapMsgRecords(records || [], columns);
+  const events = [];
+  for (const m of messages) {
+    const text = typeof m.text === "string" ? m.text : "";
+    const occurredAt = Number.isFinite(m.createdTimeMs) && m.createdTimeMs > 0
+      ? m.createdTimeMs
+      : now;
+    const key = _stableKey([
+      cfg.platform,
+      String(m.conversationId || m.senderUid || ""),
+      String(occurredAt),
+      text.slice(0, 64),
+    ]);
+    events.push({
+      id: newId(),
+      type: "event",
+      subtype: "message",
+      occurredAt,
+      actor: "person-self",
+      content: {
+        title: text ? text.slice(0, 80) : "(非文本消息)",
+        text,
+      },
+      ingestedAt: now,
+      source: {
+        adapter: cfg.sourceAdapter,
+        adapterVersion: SALVAGE_VERSION,
+        originalId: `salvage:${cfg.platform}:${key}`,
+        capturedAt: occurredAt,
+        // schema enum: export|api|sqlite|accessibility|ocr|manual. Salvaged
+        // pages ARE SQLite rows; mem-salvage provenance lives in extra.salvaged.
+        capturedBy: "sqlite",
+      },
+      extra: {
+        platform: cfg.platform,
+        channel: "im",
+        salvaged: true,
+        senderUid: m.senderUid || null,
+        conversationId: m.conversationId || null,
+        contentBlob: typeof m.contentBlob === "string" ? m.contentBlob : null,
+      },
+    });
+  }
+  return { events, mapped: messages.length, columns, sourceAdapter: cfg.sourceAdapter };
+}
+
+/**
+ * Salvage a memory dump and write the recovered messages straight into the
+ * vault with the correct per-app source. Returns counts.
+ *
+ * @param {object} vault  LocalVault (must expose putBatch)
+ * @param {string} dumpPath
+ * @param {{ app?: string, columns?: string[], unaligned?: boolean, pageSize?: number, minCols?: number, now?: number }} [opts]
+ * @returns {{ ingested: number, salvaged: number, app: string, sourceAdapter: string, leafPages: number }}
+ */
+function salvageDumpToVault(vault, dumpPath, opts = {}) {
+  if (!vault || typeof vault.putBatch !== "function") {
+    throw new TypeError("salvageDumpToVault: vault with putBatch required");
+  }
+  if (typeof dumpPath !== "string" || !dumpPath) {
+    throw new TypeError("salvageDumpToVault: dumpPath required");
+  }
+  const { records, pages } = salvageFile(dumpPath, {
+    pageSize: opts.pageSize,
+    minCols: opts.minCols,
+    unaligned: opts.unaligned,
+    stride: opts.stride,
+  });
+  const built = buildSalvageEvents(records, opts);
+  const res = built.events.length
+    ? vault.putBatch({ events: built.events })
+    : { events: 0 };
+  return {
+    ingested: res.events || 0,
+    salvaged: records.length,
+    mapped: built.mapped,
+    app: String(opts.app || "douyin").toLowerCase(),
+    sourceAdapter: built.sourceAdapter,
+    leafPages: pages,
+  };
+}
+
+module.exports = {
+  APP_SALVAGE,
+  resolveApp,
+  buildSalvageEvents,
+  salvageDumpToVault,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chainlesschain/personal-data-hub",
-  "version": "0.4.26",
+  "version": "0.4.27",
   "description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
   "type": "commonjs",
   "main": "lib/index.js",
@@ -68,7 +68,9 @@
     "./adapters/messaging-qq": "./lib/adapters/messaging-qq/index.js",
     "./adapters/messaging-telegram": "./lib/adapters/messaging-telegram/index.js",
     "./adapters/messaging-whatsapp": "./lib/adapters/messaging-whatsapp/index.js",
-    "./sidecar": "./lib/sidecar/index.js"
+    "./sidecar": "./lib/sidecar/index.js",
+    "./forensics/leaf-salvage": "./lib/forensics/leaf-salvage.js",
+    "./forensics/salvage-ingest": "./lib/forensics/salvage-ingest.js"
   },
   "scripts": {
     "test": "vitest run",