@chainlesschain/personal-data-hub 0.2.3 → 0.3.0

package/__tests__/query-parser.test.js

@@ -7,6 +7,7 @@ const {
   parseTimeWindow,
   parseFilters,
   parseIntent,
+  extractEntityTerm,
 } = require("../lib/query-parser");
 
 // Pin "now" to 2026-05-19 12:00:00 UTC for deterministic windows
@@ -148,3 +149,68 @@ describe("parseQuery (integration)", () => {
     expect(r.filters).toEqual({});
   });
 });
+
+// ─── extractEntityTerm — FTS5 fulltext routing helper ───────────────────
+//
+// 2026-05-24 — Powers AnalysisEngine._gatherFacts intent=list augmentation:
+// when the parser pulls a probable entity-name out of the question, the
+// engine appends vault.searchEvents(q=term) hits to the FACTS pool. Wrong
+// extractions are intentionally non-fatal — they waste a few rows of
+// budget at worst, never lose existing events. Memory:
+// pdh_analysis_engine_intent_routing.md.
+
+describe("extractEntityTerm", () => {
+  it("extracts named entity from '提到 X 的消息' phrasing", () => {
+    expect(extractEntityTerm("提到王老板的消息")).toBe("王老板");
+  });
+
+  it("returns null when only stop-words remain (no entity hint)", () => {
+    expect(extractEntityTerm("上个月在淘宝总共花了多少？")).toBeNull();
+    expect(extractEntityTerm("在淘宝买了什么")).toBeNull();
+  });
+
+  it("picks the longest remaining chunk when several survive cleaning", () => {
+    // 苹果(2) vs 订单(stop) — only 苹果 left.
+    expect(extractEntityTerm("苹果的订单")).toBe("苹果");
+  });
+
+  it("strips list/search trigger words ('提到', '查找', '看一下')", () => {
+    expect(extractEntityTerm("查找王医生的订单")).toBe("王医生");
+    expect(extractEntityTerm("看一下王医生的最新消息")).toBe("王医生");
+  });
+
+  it("strips compound subtype keywords before shorter intent forms", () => {
+    // "多少钱" must clear before "多少" leaves stranded "钱". With clean
+    // stripping there is no leftover ≥2 char chunk → null.
+    expect(extractEntityTerm("我总共花了多少钱")).toBeNull();
+  });
+
+  it("ignores single-character residues (verbs leak through; 1-char names skipped first-pass)", () => {
+    // "我妈" → "我" stripped (pronoun), "妈" left as single char → filtered.
+    // Documented limitation; first-pass tradeoff for higher precision.
+    expect(extractEntityTerm("我妈最近发的微信")).toBeNull();
+  });
+
+  it("handles ASCII entity tokens (≥2 chars)", () => {
+    expect(extractEntityTerm("提到 GitHub 的消息")).toBe("GitHub");
+  });
+
+  it("returns null for non-string / empty input", () => {
+    expect(extractEntityTerm("")).toBeNull();
+    expect(extractEntityTerm(null)).toBeNull();
+    expect(extractEntityTerm(undefined)).toBeNull();
+    expect(extractEntityTerm(123)).toBeNull();
+  });
+
+  it("does not pick adapter keywords as the entity (handled by filters)", () => {
+    // "淘宝" 是 adapter，会被 parseFilters 抽走当 q.adapter；不该再被
+    // 当实体名重复 FTS 搜。
+    expect(extractEntityTerm("看下淘宝的订单")).toBeNull();
+  });
+
+  it("> 10 char tokens are dropped (probable concatenated noise)", () => {
+    // 拼出一个 12 char ASCII token，期望被 length 上限过滤掉
+    const r = extractEntityTerm("提到 abcdefghijkl 的消息");
+    expect(r).toBeNull();
+  });
+});

package/__tests__/registry.test.js

@@ -304,3 +304,117 @@ describe("Phase 2 perf gate: 1k events ingest", () => {
     expect(qDur).toBeLessThan(2000); // 1k-row read should be ms-scale
   }, 60_000); // vitest test timeout — extra headroom for slow CI
 });
+
+// ─── rederive (recover orphan raw_events) ────────────────────────────────
+
+describe("AdapterRegistry.rederive", () => {
+  it("promotes raw_events to canonical events for registered adapter", async () => {
+    freshVault();
+    const reg = new AdapterRegistry({ vault });
+    const adapter = new MockAdapter({ name: "mock-rederive" });
+    reg.register(adapter);
+
+    // Simulate the failure mode: putRawEvent succeeded (raw landed) but
+    // putBatch failed at sync time (e.g. partial-index drift trap #25),
+    // so events table is empty while raw_events has 3 rows.
+    const baseTs = 1_700_000_000_000;
+    for (let i = 0; i < 3; i++) {
+      vault.putRawEvent({
+        adapter: "mock-rederive",
+        originalId: `raw-${i}`,
+        capturedAt: baseTs + i * 1000,
+        payload: {
+          variant: 1,
+          senderName: `Alice${i}`,
+          text: `hello ${i}`,
+          index: i,
+        },
+      });
+    }
+    expect(vault.stats().rawEvents).toBe(3);
+    expect(vault.stats().events).toBe(0);
+
+    const report = await reg.rederive();
+
+    expect(report.rawSeen).toBe(3);
+    expect(report.adapterMissing).toBe(0);
+    expect(report.invalidCount).toBe(0);
+    expect(report.entityCounts.events).toBe(3);
+    expect(report.entityCounts.persons).toBe(3); // MockAdapter variant 1 yields 1 person/raw
+    expect(report.errors).toEqual([]);
+    expect(vault.stats().events).toBe(3);
+    expect(vault.stats().persons).toBe(3);
+  });
+
+  it("counts raws whose adapter is not registered", async () => {
+    freshVault();
+    const reg = new AdapterRegistry({ vault });
+    // No adapter registered — every raw is orphan
+    vault.putRawEvent({
+      adapter: "ghost-adapter",
+      originalId: "x",
+      capturedAt: Date.now(),
+      payload: {},
+    });
+    const report = await reg.rederive();
+    expect(report.rawSeen).toBe(1);
+    expect(report.adapterMissing).toBe(1);
+    expect(report.entityCounts.events).toBe(0);
+    expect(vault.stats().events).toBe(0);
+  });
+
+  it("isolates per-raw normalize failures (invalidCount)", async () => {
+    freshVault();
+    const reg = new AdapterRegistry({ vault });
+    // MockAdapter.normalize throws on call #3 (1-indexed). 3 raws → 3rd throws,
+    // 1st+2nd succeed → events=2, invalidCount=1.
+    const adapter = new MockAdapter({ name: "mock-throw" });
+    adapter.normalizeShouldThrowAt(2);
+    reg.register(adapter);
+    for (let i = 0; i < 3; i++) {
+      vault.putRawEvent({
+        adapter: "mock-throw",
+        originalId: `raw-${i}`,
+        capturedAt: 1_700_000_000_000 + i * 1000,
+        payload: { variant: 1, senderName: `Bob${i}`, text: `t${i}`, index: i },
+      });
+    }
+    const report = await reg.rederive();
+    expect(report.rawSeen).toBe(3);
+    expect(report.invalidCount).toBe(1);
+    expect(report.entityCounts.events).toBe(2);
+    expect(vault.stats().events).toBe(2);
+  });
+
+  it("filters by --adapter option", async () => {
+    freshVault();
+    const reg = new AdapterRegistry({ vault });
+    reg.register(new MockAdapter({ name: "adapter-a" }));
+    reg.register(new MockAdapter({ name: "adapter-b" }));
+    vault.putRawEvent({
+      adapter: "adapter-a", originalId: "a1", capturedAt: 1_700_000_000_001,
+      payload: { variant: 1, senderName: "S", text: "ta", index: 1 },
+    });
+    vault.putRawEvent({
+      adapter: "adapter-b", originalId: "b1", capturedAt: 1_700_000_000_002,
+      payload: { variant: 1, senderName: "S", text: "tb", index: 1 },
+    });
+    const report = await reg.rederive({ adapter: "adapter-a" });
+    expect(report.rawSeen).toBe(1); // only adapter-a was queried
+    expect(report.entityCounts.events).toBe(1);
+  });
+
+  it("is idempotent: rerunning produces same events table (UPSERT via partial index)", async () => {
+    freshVault();
+    const reg = new AdapterRegistry({ vault });
+    reg.register(new MockAdapter({ name: "idemp" }));
+    vault.putRawEvent({
+      adapter: "idemp", originalId: "x", capturedAt: 1_700_000_000_000,
+      payload: { variant: 1, senderName: "Alice", text: "hi", index: 0 },
+    });
+    await reg.rederive();
+    const eventsAfterFirst = vault.stats().events;
+    await reg.rederive();
+    expect(vault.stats().events).toBe(eventsAfterFirst);
+  });
+});

package/__tests__/sidecar-contacts-cross-validate.test.js

@@ -38,7 +38,30 @@ try {
   pythonAvailable = false;
 }
 
-const describePy = pythonAvailable ? describe : describe.skip;
+// Probe bs3mc — fixture seeding opens an unencrypted SQLite file via
+// better-sqlite3-multiple-ciphers, which fails on dev boxes where the root
+// node_modules binding was compiled for Electron's NODE_MODULE_VERSION
+// instead of the host Node ABI. Skip cleanly when the native binding
+// can't load; CI Linux builds get a Node-ABI binary and runs the full path.
+let bs3mcAvailable = true;
+try {
+  const probeDir = fs.mkdtempSync(path.join(os.tmpdir(), "bs3mc-contacts-probe-"));
+  const probeDb = new Database(path.join(probeDir, "p.db"));
+  probeDb.close();
+  fs.rmSync(probeDir, { recursive: true, force: true });
+} catch (_err) {
+  bs3mcAvailable = false;
+}
+
+// FTS5 sandbox runner: the relative path to personal-data-hub-bridge
+// resolves outside the temp tree. Without this gate the spawn fails with
+// ENOENT during beforeAll instead of skipping cleanly.
+const sidecarRootAvailable = fs.existsSync(SIDECAR_ROOT);
+
+const describePy =
+  pythonAvailable && bs3mcAvailable && sidecarRootAvailable
+    ? describe
+    : describe.skip;
 
 function seedFixtureContactsDb(dbPath) {
   const db = new Database(dbPath);

package/__tests__/sidecar-supervisor.test.js

@@ -2,6 +2,7 @@
 
 import { describe, it, expect, beforeEach, afterEach } from "vitest";
 import path from "node:path";
+import fs from "node:fs";
 
 const {
   SidecarSupervisor,
@@ -46,7 +47,14 @@ try {
   pythonAvailable = false;
 }
 
-const itPy = pythonAvailable ? it : it.skip;
+// In the FTS5 sandbox runner the relative ../../personal-data-hub-bridge
+// resolves outside the temp tree and the directory does not exist; without
+// this gate, spawn() returns ENOENT and the test fails for environment
+// reasons rather than code reasons. Keep both gates so the file is safe
+// to run in either layout.
+const sidecarRootAvailable = fs.existsSync(SIDECAR_ROOT);
+
+const itPy = pythonAvailable && sidecarRootAvailable ? it : it.skip;
 
 describe("SidecarSupervisor (forensics-bridge integration)", () => {
   let supervisor;

package/__tests__/social-kuaishou-snapshot.test.js

@@ -29,9 +29,14 @@ describe("KuaishouAdapter snapshot mode", () => {
     tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "kuaishou-snap-"));
   });
 
-  it("exports SNAPSHOT_SCHEMA_VERSION = 1 + 3 VALID_SNAPSHOT_KINDS", () => {
+  it("exports SNAPSHOT_SCHEMA_VERSION = 1 + 4 VALID_SNAPSHOT_KINDS (v0.2.1 adds profile)", () => {
     expect(SNAPSHOT_SCHEMA_VERSION).toBe(1);
-    expect(VALID_SNAPSHOT_KINDS).toEqual(["watch", "collect", "search"]);
+    expect(VALID_SNAPSHOT_KINDS).toEqual([
+      "profile",
+      "watch",
+      "collect",
+      "search",
+    ]);
   });
 
   it("authenticate(inputPath) ok when readable", async () => {
@@ -100,6 +105,54 @@ describe("KuaishouAdapter snapshot mode", () => {
     expect(raws.length).toBe(0);
   });
 
+  it("v0.2 profile event normalizes to person-self with kuaishou-uid identifier", async () => {
+    const now = Date.now();
+    const p = writeSnapshot(tmpDir, {
+      schemaVersion: 1,
+      snapshottedAt: now,
+      account: { uid: "77777", displayName: "alice" },
+      events: [
+        {
+          kind: "profile",
+          id: "profile-77777",
+          capturedAt: now - 500,
+          uid: "77777",
+          nickname: "alice",
+          kuaishouId: "alice_KS",
+          avatarUrl: "https://p.kuaishou.com/u/alice.jpg",
+          sex: "F",
+          city: "Shanghai",
+          constellation: "Libra",
+          description: "hi there",
+        },
+      ],
+    });
+    const a = new KuaishouAdapter();
+    const raws = [];
+    for await (const r of a.sync({ inputPath: p })) raws.push(r);
+    expect(raws.length).toBe(1);
+    expect(raws[0].kind).toBe("profile");
+    expect(raws[0].originalId).toMatch(/^kuaishou:profile:/);
+
+    const batch = a.normalize(raws[0]);
+    expect(validateBatch(batch).valid).toBe(true);
+    expect(batch.events.length).toBe(0);
+    expect(batch.persons.length).toBe(1);
+    const person = batch.persons[0];
+    expect(person.id).toBe("person-kuaishou-77777");
+    expect(person.subtype).toBe("self");
+    expect(person.names).toEqual(["alice"]);
+    expect(person.identifiers["kuaishou-uid"]).toEqual(["77777"]);
+    expect(person.identifiers["kuaishou-id"]).toEqual(["alice_KS"]);
+    expect(person.extra.platform).toBe("kuaishou");
+    expect(person.extra.avatarUrl).toBe("https://p.kuaishou.com/u/alice.jpg");
+    expect(person.extra.sex).toBe("F");
+    expect(person.extra.city).toBe("Shanghai");
+    expect(person.extra.constellation).toBe("Libra");
+    expect(person.extra.description).toBe("hi there");
+    expect(person.source.capturedBy).toBe("api");
+  });
+
   it("watch event round-trips normalize cleanly (BROWSE subtype)", async () => {
     const now = Date.now();
     const p = writeSnapshot(tmpDir, {

package/__tests__/social-toutiao-snapshot.test.js

@@ -31,9 +31,14 @@ describe("ToutiaoAdapter snapshot mode", () => {
     tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "toutiao-snap-"));
   });
 
-  it("exports SNAPSHOT_SCHEMA_VERSION = 1 + 3 VALID_SNAPSHOT_KINDS", () => {
+  it("exports SNAPSHOT_SCHEMA_VERSION = 1 + 4 VALID_SNAPSHOT_KINDS (v0.2.1 adds profile)", () => {
     expect(SNAPSHOT_SCHEMA_VERSION).toBe(1);
-    expect(VALID_SNAPSHOT_KINDS).toEqual(["read", "collection", "search"]);
+    expect(VALID_SNAPSHOT_KINDS).toEqual([
+      "profile",
+      "read",
+      "collection",
+      "search",
+    ]);
   });
 
   it("authenticate(inputPath) ok when readable", async () => {
@@ -102,6 +107,53 @@ describe("ToutiaoAdapter snapshot mode", () => {
     expect(raws.length).toBe(0);
   });
 
+  it("v0.2 profile event normalizes to person-self with toutiao-uid identifier", async () => {
+    const now = Date.now();
+    const p = writeSnapshot(tmpDir, {
+      schemaVersion: 1,
+      snapshottedAt: now,
+      account: { uid: "99999", displayName: "alice" },
+      events: [
+        {
+          kind: "profile",
+          id: "profile-99999",
+          capturedAt: now - 500,
+          uid: "99999",
+          nickname: "alice",
+          avatarUrl: "https://p.toutiao.com/u/alice.jpg",
+          description: "hi there",
+          followingCount: 12,
+          followerCount: 34,
+          mediaId: "media-1",
+        },
+      ],
+    });
+    const a = new ToutiaoAdapter();
+    const raws = [];
+    for await (const r of a.sync({ inputPath: p })) raws.push(r);
+    expect(raws.length).toBe(1);
+    expect(raws[0].kind).toBe("profile");
+    expect(raws[0].originalId).toMatch(/^toutiao:profile:/);
+
+    const batch = a.normalize(raws[0]);
+    expect(validateBatch(batch).valid).toBe(true);
+    // KIND_PROFILE produces a person record (not an event)
+    expect(batch.events.length).toBe(0);
+    expect(batch.persons.length).toBe(1);
+    const person = batch.persons[0];
+    expect(person.id).toBe("person-toutiao-99999");
+    expect(person.subtype).toBe("self");
+    expect(person.names).toEqual(["alice"]);
+    expect(person.identifiers["toutiao-uid"]).toEqual(["99999"]);
+    expect(person.identifiers["toutiao-media-id"]).toEqual(["media-1"]);
+    expect(person.extra.platform).toBe("toutiao");
+    expect(person.extra.avatarUrl).toBe("https://p.toutiao.com/u/alice.jpg");
+    expect(person.extra.description).toBe("hi there");
+    expect(person.extra.followingCount).toBe(12);
+    expect(person.extra.followerCount).toBe(34);
+    expect(person.source.capturedBy).toBe("api");
+  });
+
   it("read event round-trips normalize cleanly (BROWSE subtype)", async () => {
     const now = Date.now();
     const p = writeSnapshot(tmpDir, {

package/__tests__/vault-search-helpers.test.js

@@ -0,0 +1,104 @@
+"use strict";
+
+/**
+ * Pure-JS unit tests for the search SQL fragment builders in vault.js.
+ * Runs without the native bs3mc binding so it works on any Node ABI
+ * (the integration-level tests in vault-search.test.js need a working
+ * native binding and only run in CI).
+ */
+
+import { describe, it, expect } from "vitest";
+
+const { _searchHelpers } = require("../lib/vault");
+const { _categoryToWhere, _quoteFtsQuery, FTS5_MIN_QUERY_LEN } = _searchHelpers;
+
+describe("_quoteFtsQuery", () => {
+  it("wraps a plain string in FTS5 phrase quotes", () => {
+    expect(_quoteFtsQuery("hello")).toBe('"hello"');
+  });
+
+  it("doubles embedded double quotes (FTS5 escape rule)", () => {
+    expect(_quoteFtsQuery('he said "hi"')).toBe('"he said ""hi"""');
+  });
+
+  it("preserves CJK as-is", () => {
+    expect(_quoteFtsQuery("支付宝订单")).toBe('"支付宝订单"');
+  });
+
+  it("does not interpret FTS5 operators (OR/AND/NEAR — they're inside quotes)", () => {
+    // The whole input is wrapped in phrase quotes, so OR is literal text
+    expect(_quoteFtsQuery("a OR b")).toBe('"a OR b"');
+  });
+
+  it("FTS5_MIN_QUERY_LEN matches trigram tokenizer requirement", () => {
+    expect(FTS5_MIN_QUERY_LEN).toBe(3);
+  });
+});
+
+describe("_categoryToWhere", () => {
+  it("returns sql=null for empty/null category (caller skips filter)", () => {
+    expect(_categoryToWhere(null)).toEqual({ sql: null, params: {} });
+    expect(_categoryToWhere("")).toEqual({ sql: null, params: {} });
+    expect(_categoryToWhere(undefined)).toEqual({ sql: null, params: {} });
+  });
+
+  it("returns '0=1' for unknown category (no rules match)", () => {
+    const r = _categoryToWhere("not-a-real-category");
+    expect(r.sql).toBe("0=1");
+    expect(r.params).toEqual({});
+  });
+
+  it("translates 'chat' to wechat-exact + messaging-prefix OR", () => {
+    const r = _categoryToWhere("chat");
+    expect(r.sql).toMatch(/source_adapter = @cat\d/);
+    expect(r.sql).toMatch(/source_adapter LIKE @cat\d/);
+    expect(Object.values(r.params)).toEqual(
+      expect.arrayContaining(["wechat", "messaging-%"])
+    );
+  });
+
+  it("translates 'social' to a single LIKE 'social-%' condition", () => {
+    const r = _categoryToWhere("social");
+    expect(r.sql).toMatch(/source_adapter LIKE @cat0/);
+    expect(r.params).toEqual({ cat0: "social-%" });
+  });
+
+  it("translates 'shopping' to shopping-% OR alipay-%", () => {
+    const r = _categoryToWhere("shopping");
+    const vals = Object.values(r.params).sort();
+    expect(vals).toEqual(["alipay-%", "shopping-%"]);
+    expect(r.sql).toMatch(/OR/);
+  });
+
+  it("translates 'system' to system-data + browser-history + local-source adapters", () => {
+    const r = _categoryToWhere("system");
+    // 7 rules currently map to "system" (see lib/categories.js PREFIX_RULES):
+    //   prefix wildcards: system-data*, browser-*
+    //   exact names:      vscode, win-recent, git-activity, shell-history, local-files
+    // If a future adapter joins this bucket, append it here.
+    const vals = Object.values(r.params).sort();
+    expect(vals).toEqual([
+      "browser-%",
+      "git-activity",
+      "local-files",
+      "shell-history",
+      "system-data%",
+      "vscode",
+      "win-recent",
+    ]);
+    expect(r.sql).toContain("OR");
+  });
+
+  it("translates 'other' to NOT-IN-any-prefix (negation form)", () => {
+    const r = _categoryToWhere("other");
+    expect(r.sql).toMatch(/NOT LIKE @cat\d/);
+    expect(r.sql).toMatch(/AND/);
+    // Should reference every prefix rule
+    expect(Object.keys(r.params).length).toBeGreaterThan(5);
+  });
+
+  it("supports custom param prefix to avoid collisions in compound queries", () => {
+    const r = _categoryToWhere("social", "xx");
+    expect(Object.keys(r.params)[0]).toMatch(/^xx/);
+  });
+});