@chainlesschain/personal-data-hub 0.4.29 → 0.4.31

package/__tests__/registry.test.js

@@ -1,420 +0,0 @@
-"use strict";
-
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-
-const fs = require("node:fs");
-const os = require("node:os");
-const path = require("node:path");
-
-const { LocalVault } = require("../lib/vault");
-const { generateKeyHex } = require("../lib/key-providers");
-const { AdapterRegistry } = require("../lib/registry");
-const { MockAdapter } = require("../lib/mock-adapter");
-
-// ─── Scaffolding ─────────────────────────────────────────────────────────
-
-let tmpDir;
-let vault;
-
-function freshVault() {
-  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "pdh-reg-"));
-  vault = new LocalVault({
-    path: path.join(tmpDir, "vault.db"),
-    key: generateKeyHex(),
-    skipAudit: true,
-  });
-  vault.open();
-}
-
-afterEach(() => {
-  if (vault) {
-    try { vault.close(); } catch (_e) {}
-    vault = null;
-  }
-  if (tmpDir && fs.existsSync(tmpDir)) {
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  }
-});
-
-// ─── Registration ────────────────────────────────────────────────────────
-
-describe("AdapterRegistry registration", () => {
-  it("rejects construction without vault", () => {
-    expect(() => new AdapterRegistry({})).toThrow(/vault/);
-    expect(() => new AdapterRegistry()).toThrow();
-  });
-
-  it("registers + lists + looks up adapters", () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    const a = new MockAdapter({ name: "mock-a" });
-    const b = new MockAdapter({ name: "mock-b" });
-    reg.register(a);
-    reg.register(b);
-    expect(reg.has("mock-a")).toBe(true);
-    expect(reg.list().map((x) => x.name).sort()).toEqual(["mock-a", "mock-b"]);
-    expect(reg.get("mock-a")).toBe(a);
-    expect(reg.get("unknown")).toBeNull();
-  });
-
-  it("rejects double-register of same name", () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ name: "dup" }));
-    expect(() => reg.register(new MockAdapter({ name: "dup" }))).toThrow(/already registered/);
-  });
-
-  it("rejects malformed adapter (assertAdapter gate)", () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    expect(() => reg.register({ name: "broken" })).toThrow(/invalid adapter/);
-  });
-
-  it("unregister removes; second call returns false", () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ name: "x" }));
-    expect(reg.unregister("x")).toBe(true);
-    expect(reg.unregister("x")).toBe(false);
-    expect(reg.has("x")).toBe(false);
-  });
-});
-
-// ─── End-to-end sync ─────────────────────────────────────────────────────
-
-describe("AdapterRegistry.syncAdapter", () => {
-  it("runs a clean MockAdapter end-to-end: vault + watermark + report", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ count: 30, seed: 1 }));
-
-    const report = await reg.syncAdapter("mock");
-
-    expect(report.status).toBe("ok");
-    expect(report.rawCount).toBe(30);
-    expect(report.invalidCount).toBe(0);
-    expect(report.entityCounts.events).toBe(30);
-    // ~2/3 of variants have a person (variants 1 and 2)
-    expect(report.entityCounts.persons).toBeGreaterThan(0);
-    expect(report.error).toBeNull();
-    expect(report.watermark).toBe("30");
-    expect(report.durationMs).toBeGreaterThan(0);
-
-    // Vault was written
-    expect(vault.stats().events).toBe(30);
-    expect(vault.stats().rawEvents).toBe(30);
-
-    // Watermark stored
-    const wm = vault.getWatermark("mock");
-    expect(wm.watermark).toBe("30");
-    expect(wm.last_status).toBe("ok");
-  });
-
-  it("incremental sync skips already-seen items via stored watermark", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ count: 50, seed: 1 }));
-
-    const first = await reg.syncAdapter("mock", { maxEvents: 20 });
-    expect(first.rawCount).toBe(20);
-    expect(first.watermark).toBe("20");
-
-    const second = await reg.syncAdapter("mock");
-    expect(second.rawCount).toBe(30); // 50 - 20
-    expect(second.watermark).toBe("50");
-    expect(vault.stats().rawEvents).toBe(50);
-  });
-
-  it("unhealthy adapter aborts before sync; records audit", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    const a = new MockAdapter({ count: 10 });
-    a.shouldFailHealth = true;
-    reg.register(a);
-
-    const report = await reg.syncAdapter("mock");
-    expect(report.status).toBe("unhealthy");
-    expect(report.rawCount).toBe(0);
-    expect(report.error).toBeTruthy();
-    expect(vault.stats().events).toBe(0);
-
-    const audits = vault.queryAudit({ action: "adapter.sync.unhealthy" });
-    expect(audits.length).toBe(1);
-  });
-
-  it("normalize failure on one item does NOT abort the whole sync", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    const a = new MockAdapter({ count: 10 });
-    a.normalizeShouldThrowAt(4); // throw on the 5th normalize call
-    reg.register(a);
-
-    const report = await reg.syncAdapter("mock");
-    expect(report.status).toBe("ok");
-    expect(report.rawCount).toBe(10);
-    expect(report.invalidCount).toBeGreaterThanOrEqual(1);
-    expect(report.entityCounts.events).toBeLessThan(10);
-
-    const audits = vault.queryAudit({ action: "adapter.sync.normalize_failed" });
-    expect(audits.length).toBe(1);
-  });
-
-  it("mid-sync throw is captured as status=error with preserved watermark", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    const a = new MockAdapter({ count: 100 });
-    a.failAfter = 25;
-    reg.register(a);
-
-    const report = await reg.syncAdapter("mock");
-    expect(report.status).toBe("error");
-    expect(report.error).toContain("induced sync failure");
-
-    const wm = vault.getWatermark("mock");
-    expect(wm.last_status).toBe("error");
-    expect(wm.last_error).toContain("induced sync failure");
-  });
-
-  it("refuses concurrent sync of two adapters in one registry", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    // Use 500 events (not 5000) — still big enough to be mid-flight when
-    // the second syncAdapter() lands, but fits comfortably in 10s.
-    reg.register(new MockAdapter({ name: "a", count: 500 }));
-    reg.register(new MockAdapter({ name: "b", count: 5 }));
-
-    const p1 = reg.syncAdapter("a");
-    // Should reject during the brief async window where a is mid-sync.
-    let racedReject = null;
-    try {
-      await reg.syncAdapter("b");
-    } catch (e) {
-      racedReject = e;
-    }
-    await p1;
-    // Depending on event-loop timing this might race the other way; we just
-    // assert no double-sync corruption. The active-sync invariant is
-    // additionally enforced by the activeSync flag.
-    expect(racedReject == null || /already syncing/.test(racedReject.message)).toBe(true);
-  }, 30_000);
-});
-
-// ─── KG + RAG sinks ──────────────────────────────────────────────────────
-
-describe("AdapterRegistry pluggable sinks", () => {
-  it("kgSink receives triples per batch; ragSink receives docs per batch", async () => {
-    freshVault();
-    const kgTriples = [];
-    const ragDocs = [];
-    const reg = new AdapterRegistry({
-      vault,
-      kgSink: (ts) => kgTriples.push(...ts),
-      ragSink: (ds) => ragDocs.push(...ds),
-      batchSize: 7, // forces multiple batches across 20 events
-    });
-    reg.register(new MockAdapter({ count: 20, seed: 1 }));
-
-    const report = await reg.syncAdapter("mock");
-    expect(report.status).toBe("ok");
-    expect(kgTriples.length).toBeGreaterThan(20); // each event yields multiple triples
-    expect(ragDocs.length).toBeGreaterThan(0);
-    // All docs should reference the mock adapter
-    expect(ragDocs.every((d) => d.metadata.adapter === "mock")).toBe(true);
-    // Triple counts in the report should match what was actually sent
-    expect(report.kgTripleCount).toBe(kgTriples.length);
-    expect(report.ragDocCount).toBe(ragDocs.length);
-  });
-
-  it("sink throws don't abort sync; failure recorded as audit", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({
-      vault,
-      kgSink: () => { throw new Error("downstream KG died"); },
-      ragSink: () => { throw new Error("downstream RAG died"); },
-    });
-    reg.register(new MockAdapter({ count: 3 }));
-
-    const report = await reg.syncAdapter("mock");
-    expect(report.status).toBe("ok");
-    expect(report.rawCount).toBe(3);
-    expect(report.entityCounts.events).toBe(3);
-
-    const kgAudits = vault.queryAudit({ action: "adapter.sync.kg_sink_failed" });
-    const ragAudits = vault.queryAudit({ action: "adapter.sync.rag_sink_failed" });
-    expect(kgAudits.length).toBeGreaterThan(0);
-    expect(ragAudits.length).toBeGreaterThan(0);
-  });
-});
-
-// ─── syncAll ─────────────────────────────────────────────────────────────
-
-describe("AdapterRegistry.syncAll", () => {
-  it("returns one report per registered adapter, isolates failures", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ name: "ok-a", count: 5 }));
-    const bad = new MockAdapter({ name: "bad", count: 10 });
-    bad.shouldFailHealth = true;
-    reg.register(bad);
-    reg.register(new MockAdapter({ name: "ok-b", count: 7 }));
-
-    const reports = await reg.syncAll();
-    expect(reports.length).toBe(3);
-    const byName = Object.fromEntries(reports.map((r) => [r.adapter, r]));
-    expect(byName["ok-a"].status).toBe("ok");
-    expect(byName["bad"].status).toBe("unhealthy");
-    expect(byName["ok-b"].status).toBe("ok");
-
-    // The bad adapter's failure didn't prevent the others' data from landing.
-    expect(vault.stats().events).toBe(12);
-  });
-});
-
-// ─── 1k events < 30s perf gate (architecture doc §15.2) ──────────────────
-
-describe("Phase 2 perf gate: 1k events ingest", () => {
-  it("ingests 1k mock events well under the 30s budget", async () => {
-    freshVault();
-    let kgCount = 0;
-    let ragCount = 0;
-    const reg = new AdapterRegistry({
-      vault,
-      kgSink: (ts) => { kgCount += ts.length; },
-      ragSink: (ds) => { ragCount += ds.length; },
-      batchSize: 200,
-    });
-    reg.register(new MockAdapter({ count: 1000, seed: 7 }));
-
-    const start = Date.now();
-    const report = await reg.syncAdapter("mock");
-    const dur = Date.now() - start;
-
-    expect(report.status).toBe("ok");
-    expect(report.rawCount).toBe(1000);
-    expect(report.entityCounts.events).toBe(1000);
-    expect(kgCount).toBeGreaterThan(1000);
-    expect(ragCount).toBeGreaterThan(0);
-    expect(dur).toBeLessThan(30_000); // architecture doc target
-
-    // Sanity probe: querying back ~1k events should be fast.
-    const qStart = Date.now();
-    const events = vault.queryEvents({ limit: 1000 });
-    const qDur = Date.now() - qStart;
-    expect(events.length).toBe(1000);
-    expect(qDur).toBeLessThan(2000); // 1k-row read should be ms-scale
-  }, 60_000); // vitest test timeout — extra headroom for slow CI
-});
-
-// ─── rederive (recover orphan raw_events) ────────────────────────────────
-
-describe("AdapterRegistry.rederive", () => {
-  it("promotes raw_events to canonical events for registered adapter", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    const adapter = new MockAdapter({ name: "mock-rederive" });
-    reg.register(adapter);
-
-    // Simulate the failure mode: putRawEvent succeeded (raw landed) but
-    // putBatch failed at sync time (e.g. partial-index drift trap #25),
-    // so events table is empty while raw_events has 3 rows.
-    const baseTs = 1_700_000_000_000;
-    for (let i = 0; i < 3; i++) {
-      vault.putRawEvent({
-        adapter: "mock-rederive",
-        originalId: `raw-${i}`,
-        capturedAt: baseTs + i * 1000,
-        payload: {
-          variant: 1,
-          senderName: `Alice${i}`,
-          text: `hello ${i}`,
-          index: i,
-        },
-      });
-    }
-    expect(vault.stats().rawEvents).toBe(3);
-    expect(vault.stats().events).toBe(0);
-
-    const report = await reg.rederive();
-
-    expect(report.rawSeen).toBe(3);
-    expect(report.adapterMissing).toBe(0);
-    expect(report.invalidCount).toBe(0);
-    expect(report.entityCounts.events).toBe(3);
-    expect(report.entityCounts.persons).toBe(3); // MockAdapter variant 1 yields 1 person/raw
-    expect(report.errors).toEqual([]);
-    expect(vault.stats().events).toBe(3);
-    expect(vault.stats().persons).toBe(3);
-  });
-
-  it("counts raws whose adapter is not registered", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    // No adapter registered — every raw is orphan
-    vault.putRawEvent({
-      adapter: "ghost-adapter",
-      originalId: "x",
-      capturedAt: Date.now(),
-      payload: {},
-    });
-    const report = await reg.rederive();
-    expect(report.rawSeen).toBe(1);
-    expect(report.adapterMissing).toBe(1);
-    expect(report.entityCounts.events).toBe(0);
-    expect(vault.stats().events).toBe(0);
-  });
-
-  it("isolates per-raw normalize failures (invalidCount)", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    // MockAdapter.normalize throws on call #3 (1-indexed). 3 raws → 3rd throws,
-    // 1st+2nd succeed → events=2, invalidCount=1.
-    const adapter = new MockAdapter({ name: "mock-throw" });
-    adapter.normalizeShouldThrowAt(2);
-    reg.register(adapter);
-    for (let i = 0; i < 3; i++) {
-      vault.putRawEvent({
-        adapter: "mock-throw",
-        originalId: `raw-${i}`,
-        capturedAt: 1_700_000_000_000 + i * 1000,
-        payload: { variant: 1, senderName: `Bob${i}`, text: `t${i}`, index: i },
-      });
-    }
-    const report = await reg.rederive();
-    expect(report.rawSeen).toBe(3);
-    expect(report.invalidCount).toBe(1);
-    expect(report.entityCounts.events).toBe(2);
-    expect(vault.stats().events).toBe(2);
-  });
-
-  it("filters by --adapter option", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ name: "adapter-a" }));
-    reg.register(new MockAdapter({ name: "adapter-b" }));
-    vault.putRawEvent({
-      adapter: "adapter-a", originalId: "a1", capturedAt: 1_700_000_000_001,
-      payload: { variant: 1, senderName: "S", text: "ta", index: 1 },
-    });
-    vault.putRawEvent({
-      adapter: "adapter-b", originalId: "b1", capturedAt: 1_700_000_000_002,
-      payload: { variant: 1, senderName: "S", text: "tb", index: 1 },
-    });
-    const report = await reg.rederive({ adapter: "adapter-a" });
-    expect(report.rawSeen).toBe(1); // only adapter-a was queried
-    expect(report.entityCounts.events).toBe(1);
-  });
-
-  it("is idempotent: rerunning produces same events table (UPSERT via partial index)", async () => {
-    freshVault();
-    const reg = new AdapterRegistry({ vault });
-    reg.register(new MockAdapter({ name: "idemp" }));
-    vault.putRawEvent({
-      adapter: "idemp", originalId: "x", capturedAt: 1_700_000_000_000,
-      payload: { variant: 1, senderName: "Alice", text: "hi", index: 0 },
-    });
-    await reg.rederive();
-    const eventsAfterFirst = vault.stats().events;
-    await reg.rederive();
-    expect(vault.stats().events).toBe(eventsAfterFirst);
-  });
-});

package/__tests__/salvage-ingest.test.js

@@ -1,97 +0,0 @@
-"use strict";
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-
-const fs = require("node:fs");
-const path = require("node:path");
-const os = require("node:os");
-
-const { LocalVault } = require("../lib/vault");
-const { generateKeyHex } = require("../lib/key-providers");
-const {
-  buildSalvageEvents,
-  salvageDumpToVault,
-  resolveApp,
-} = require("../lib/forensics/salvage-ingest");
-
-// Build a real SQLite DB and treat its bytes as a memory dump; verify the
-// generic salvage→vault path recovers messages AND tags them with the correct
-// per-app source.adapter (multi-app de-silo). Real LocalVault → proves the
-// hand-built events pass schema validation + are searchable.
-describe("salvage-ingest — generic multi-app salvage → vault", () => {
-  let dir, dumpPath, vault, vdir;
-  const COLUMNS = ["msg_uuid", "conversation_id", "sender", "content", "created_time"];
-
-  beforeAll(() => {
-    const Database = require("better-sqlite3-multiple-ciphers");
-    dir = fs.mkdtempSync(path.join(os.tmpdir(), "salvage-ing-"));
-    dumpPath = path.join(dir, "u.db");
-    const db = new Database(dumpPath);
-    db.exec("CREATE TABLE msg(msg_uuid TEXT, conversation_id TEXT, sender INTEGER, content TEXT, created_time INTEGER)");
-    const ins = db.prepare("INSERT INTO msg VALUES(?,?,?,?,?)");
-    ins.run("u1", "conv-1", 111, "今天的会议改到下午三点 hi", 1700000000000);
-    ins.run("u2", "conv-1", 222, "收到 👌", 1700000001000);
-    db.close();
-
-    vdir = fs.mkdtempSync(path.join(os.tmpdir(), "salvage-vault-"));
-    vault = new LocalVault({ path: path.join(vdir, "v.db"), key: generateKeyHex() });
-    vault.open();
-  });
-
-  afterAll(() => {
-    try { vault.close(); } catch (_e) {}
-    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) {}
-    try { fs.rmSync(vdir, { recursive: true, force: true }); } catch (_e) {}
-  });
-
-  it("resolveApp maps known apps to canonical source adapters; unknown → salvage:<app>", () => {
-    expect(resolveApp("douyin").sourceAdapter).toBe("social-douyin");
-    expect(resolveApp("toutiao").sourceAdapter).toBe("social-toutiao");
-    expect(resolveApp("wechat").sourceAdapter).toBe("wechat");
-    expect(resolveApp("kuaishou").sourceAdapter).toBe("social-kuaishou");
-    expect(resolveApp("bogusapp").sourceAdapter).toBe("salvage:bogusapp");
-  });
-
-  it("buildSalvageEvents tags per-app source + builds valid message events", () => {
-    const recs = [{ rowid: "1", cols: ["u1", "conv-1", 111, "hello 世界", 1700000000000] }];
-    const built = buildSalvageEvents(recs, { app: "toutiao", columns: COLUMNS, now: 1700000099000 });
-    expect(built.events.length).toBe(1);
-    const e = built.events[0];
-    expect(e.source.adapter).toBe("social-toutiao");
-    expect(e.source.capturedBy).toBe("sqlite"); // schema enum; provenance in extra.salvaged
-    expect(e.subtype).toBe("message");
-    expect(e.content.text).toBe("hello 世界");
-    expect(e.extra.platform).toBe("toutiao");
-    expect(e.extra.salvaged).toBe(true);
-  });
-
-  it("salvageDumpToVault ingests with douyin source + events are searchable", () => {
-    const r = salvageDumpToVault(vault, dumpPath, { app: "douyin", columns: COLUMNS, now: 1700000099000 });
-    expect(r.app).toBe("douyin");
-    expect(r.sourceAdapter).toBe("social-douyin");
-    expect(r.ingested).toBe(2);
-    // events landed under the correct source + are searchable
-    const events = vault.queryEvents({ limit: 100 }) || [];
-    const douyin = events.filter((e) => e.source && e.source.adapter === "social-douyin");
-    expect(douyin.length).toBe(2);
-    const texts = douyin.map((e) => e.content && e.content.text).sort();
-    expect(texts).toContain("收到 👌"); // UTF-8 emoji survives
-  });
-
-  it("same dump under a different app tags a different source (no cross-attribution)", () => {
-    const r = salvageDumpToVault(vault, dumpPath, { app: "toutiao", columns: COLUMNS, now: 1700000099000 });
-    expect(r.sourceAdapter).toBe("social-toutiao");
-    expect(r.ingested).toBe(2);
-    const events = vault.queryEvents({ limit: 100 }) || [];
-    expect(events.filter((e) => e.source && e.source.adapter === "social-toutiao").length).toBe(2);
-    // douyin events from prior test remain distinct
-    expect(events.filter((e) => e.source && e.source.adapter === "social-douyin").length).toBe(2);
-  });
-
-  it("re-ingesting the same dump dedups (stable originalId)", () => {
-    const before = (vault.queryEvents({ limit: 200 }) || []).length;
-    salvageDumpToVault(vault, dumpPath, { app: "douyin", columns: COLUMNS, now: 1700000099000 });
-    const after = (vault.queryEvents({ limit: 200 }) || []).length;
-    expect(after).toBe(before); // ON CONFLICT(source_adapter, source_original_id) updates, no dupes
-  });
-});