@chainlesschain/personal-data-hub 0.4.28 → 0.4.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/README.md +13 -5
  2. package/lib/adapters/social-douyin-adb/usage-profile-reader.js +253 -0
  3. package/lib/adapters/social-douyin-adb/watch-history-reader.js +104 -31
  4. package/lib/adapters/social-toutiao-adb/article-reader.js +202 -0
  5. package/lib/analysis-skills/overview.js +24 -4
  6. package/lib/analysis-skills/spending.js +63 -2
  7. package/lib/analysis-skills/timeline.js +11 -6
  8. package/lib/prompt-builder.js +15 -1
  9. package/lib/query-parser.js +38 -8
  10. package/package.json +4 -1
  11. package/__tests__/adapter-guide.test.js +0 -47
  12. package/__tests__/adapter-spec.test.js +0 -78
  13. package/__tests__/adapters/ai-chat-cookie-capture-spec.test.js +0 -211
  14. package/__tests__/adapters/ai-chat-health-checker.test.js +0 -262
  15. package/__tests__/adapters/ai-chat-history.test.js +0 -396
  16. package/__tests__/adapters/ai-chat-http-client.test.js +0 -242
  17. package/__tests__/adapters/ai-chat-vendors.test.js +0 -874
  18. package/__tests__/adapters/alipay-bill-adapter.test.js +0 -538
  19. package/__tests__/adapters/apple-health.test.js +0 -95
  20. package/__tests__/adapters/bank-family.test.js +0 -125
  21. package/__tests__/adapters/biz-tianyancha.test.js +0 -159
  22. package/__tests__/adapters/browser-history-chrome.test.js +0 -377
  23. package/__tests__/adapters/browser-history-edge.test.js +0 -159
  24. package/__tests__/adapters/car-mercedesme.test.js +0 -74
  25. package/__tests__/adapters/doc-baidu-netdisk.test.js +0 -102
  26. package/__tests__/adapters/doc-camscanner.test.js +0 -147
  27. package/__tests__/adapters/doc-platforms.test.js +0 -177
  28. package/__tests__/adapters/edu-huawei-learning-live.test.js +0 -198
  29. package/__tests__/adapters/edu-zuoyebang-live.test.js +0 -226
  30. package/__tests__/adapters/email-adapter-snapshot.test.js +0 -237
  31. package/__tests__/adapters/email-adapter.test.js +0 -742
  32. package/__tests__/adapters/email-classifier.test.js +0 -347
  33. package/__tests__/adapters/email-imap-session.test.js +0 -334
  34. package/__tests__/adapters/email-parser.test.js +0 -244
  35. package/__tests__/adapters/email-pdf-extractor.test.js +0 -529
  36. package/__tests__/adapters/email-providers.test.js +0 -84
  37. package/__tests__/adapters/email-retry-progress.test.js +0 -294
  38. package/__tests__/adapters/email-templates.test.js +0 -822
  39. package/__tests__/adapters/family-23-collectors-scaffold.test.js +0 -182
  40. package/__tests__/adapters/finance-alipay-live.test.js +0 -258
  41. package/__tests__/adapters/finance-dcep.test.js +0 -74
  42. package/__tests__/adapters/fitness-joyrun.test.js +0 -82
  43. package/__tests__/adapters/game-genshin-live.test.js +0 -238
  44. package/__tests__/adapters/game-genshin-scaffold.test.js +0 -108
  45. package/__tests__/adapters/game-honor-of-kings-live.test.js +0 -230
  46. package/__tests__/adapters/git-activity.test.js +0 -222
  47. package/__tests__/adapters/gov-12123.test.js +0 -103
  48. package/__tests__/adapters/gov-ixiamen.test.js +0 -150
  49. package/__tests__/adapters/gov-tax.test.js +0 -135
  50. package/__tests__/adapters/health-meiyou.test.js +0 -125
  51. package/__tests__/adapters/local-files.test.js +0 -264
  52. package/__tests__/adapters/local-im-pc.test.js +0 -154
  53. package/__tests__/adapters/messaging-whatsapp.test.js +0 -289
  54. package/__tests__/adapters/music-kugou.test.js +0 -187
  55. package/__tests__/adapters/music-qq.test.js +0 -112
  56. package/__tests__/adapters/netease-music-live.test.js +0 -244
  57. package/__tests__/adapters/netease-music.test.js +0 -74
  58. package/__tests__/adapters/pc-local-discovery.test.js +0 -141
  59. package/__tests__/adapters/qq-pc-direct-read.test.js +0 -227
  60. package/__tests__/adapters/reading-family.test.js +0 -108
  61. package/__tests__/adapters/recruit-boss.test.js +0 -180
  62. package/__tests__/adapters/shell-history.test.js +0 -180
  63. package/__tests__/adapters/shopping-base.test.js +0 -179
  64. package/__tests__/adapters/shopping-dianping.test.js +0 -239
  65. package/__tests__/adapters/social-bilibili-adb-api-client.test.js +0 -721
  66. package/__tests__/adapters/social-bilibili-adb-chromium-cookies-reader.test.js +0 -346
  67. package/__tests__/adapters/social-bilibili-adb-collector.test.js +0 -284
  68. package/__tests__/adapters/social-bilibili-adb-cookies-extension.test.js +0 -343
  69. package/__tests__/adapters/social-bilibili-adb-snapshot-builder.test.js +0 -296
  70. package/__tests__/adapters/social-csdn.test.js +0 -175
  71. package/__tests__/adapters/social-dongchedi.test.js +0 -165
  72. package/__tests__/adapters/social-douyin-adb-aweme-detail.test.js +0 -165
  73. package/__tests__/adapters/social-douyin-adb-collector.test.js +0 -254
  74. package/__tests__/adapters/social-douyin-adb-db-extension.test.js +0 -114
  75. package/__tests__/adapters/social-douyin-adb-im-db-parser.test.js +0 -304
  76. package/__tests__/adapters/social-douyin-adb-snapshot-builder.test.js +0 -216
  77. package/__tests__/adapters/social-douyin-adb-watch-history.test.js +0 -192
  78. package/__tests__/adapters/social-kuaishou-adb-api-client.test.js +0 -496
  79. package/__tests__/adapters/social-kuaishou-adb-collector.test.js +0 -276
  80. package/__tests__/adapters/social-kuaishou-adb-cookies-extension.test.js +0 -152
  81. package/__tests__/adapters/social-kuaishou-adb-snapshot-builder.test.js +0 -178
  82. package/__tests__/adapters/social-toutiao-adb-account-reader.test.js +0 -135
  83. package/__tests__/adapters/social-toutiao-adb-api-client.test.js +0 -626
  84. package/__tests__/adapters/social-toutiao-adb-collector.test.js +0 -378
  85. package/__tests__/adapters/social-toutiao-adb-cookies-extension.test.js +0 -193
  86. package/__tests__/adapters/social-toutiao-adb-snapshot-builder.test.js +0 -196
  87. package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +0 -311
  88. package/__tests__/adapters/social-weibo-adb-api-client.test.js +0 -362
  89. package/__tests__/adapters/social-weibo-adb-collector.test.js +0 -201
  90. package/__tests__/adapters/social-weibo-adb-cookies-extension.test.js +0 -167
  91. package/__tests__/adapters/social-weibo-adb-snapshot-builder.test.js +0 -189
  92. package/__tests__/adapters/social-xiaohongshu-adb-api-client.test.js +0 -431
  93. package/__tests__/adapters/social-xiaohongshu-adb-collector.test.js +0 -207
  94. package/__tests__/adapters/social-xiaohongshu-adb-cookies-extension.test.js +0 -0
  95. package/__tests__/adapters/social-xiaohongshu-adb-sign-provider-injection.test.js +0 -351
  96. package/__tests__/adapters/social-xiaohongshu-adb-sign.test.js +0 -130
  97. package/__tests__/adapters/social-xiaohongshu-adb-snapshot-builder.test.js +0 -200
  98. package/__tests__/adapters/social-zhihu.test.js +0 -246
  99. package/__tests__/adapters/system-data-adapter.test.js +0 -443
  100. package/__tests__/adapters/system-data-android-ingest.test.js +0 -144
  101. package/__tests__/adapters/system-data-android.test.js +0 -519
  102. package/__tests__/adapters/system-data-disclosure.test.js +0 -153
  103. package/__tests__/adapters/travel-12306.test.js +0 -512
  104. package/__tests__/adapters/travel-amap.test.js +0 -219
  105. package/__tests__/adapters/travel-baidu-map.test.js +0 -305
  106. package/__tests__/adapters/travel-base.test.js +0 -205
  107. package/__tests__/adapters/travel-ctrip.test.js +0 -377
  108. package/__tests__/adapters/travel-didi-consumer.test.js +0 -66
  109. package/__tests__/adapters/travel-didi.test.js +0 -204
  110. package/__tests__/adapters/travel-tencent-map.test.js +0 -207
  111. package/__tests__/adapters/travel-tongcheng.test.js +0 -289
  112. package/__tests__/adapters/video-platforms.test.js +0 -152
  113. package/__tests__/adapters/video-xigua.test.js +0 -106
  114. package/__tests__/adapters/vscode.test.js +0 -299
  115. package/__tests__/adapters/wechat-bootstrap.test.js +0 -240
  116. package/__tests__/adapters/wechat-env-probe.test.js +0 -162
  117. package/__tests__/adapters/wechat-frida-agent.test.js +0 -322
  118. package/__tests__/adapters/wechat-frida-integration.test.js +0 -149
  119. package/__tests__/adapters/wechat-frida-key-provider.test.js +0 -188
  120. package/__tests__/adapters/wechat-md5-key-provider.test.js +0 -101
  121. package/__tests__/adapters/wechat-pc-direct-read.test.js +0 -365
  122. package/__tests__/adapters/wechat-pc-group-topic.test.js +0 -63
  123. package/__tests__/adapters/wechat-pc-v4-sidecar.test.js +0 -72
  124. package/__tests__/adapters/weread.test.js +0 -123
  125. package/__tests__/adapters/wework-pc.test.js +0 -124
  126. package/__tests__/adapters/win-recent.test.js +0 -192
  127. package/__tests__/analysis-skills.test.js +0 -679
  128. package/__tests__/analysis.test.js +0 -1845
  129. package/__tests__/audio-ximalaya-snapshot.test.js +0 -279
  130. package/__tests__/batch.test.js +0 -133
  131. package/__tests__/bridges-cc-kg.test.js +0 -231
  132. package/__tests__/bridges-cc-llm.test.js +0 -191
  133. package/__tests__/bridges-cc-rag.test.js +0 -162
  134. package/__tests__/categories.test.js +0 -92
  135. package/__tests__/e2e/ai-chat-cross-source-journey.test.js +0 -213
  136. package/__tests__/e2e/full-user-journey.test.js +0 -188
  137. package/__tests__/e2e/local-data-adapters-cli.e2e.test.js +0 -146
  138. package/__tests__/entity-resolver-ingest-hook.test.js +0 -177
  139. package/__tests__/entity-resolver-stages.test.js +0 -411
  140. package/__tests__/entity-resolver-vault.test.js +0 -249
  141. package/__tests__/entity-resolver.test.js +0 -526
  142. package/__tests__/fitness-keep-snapshot.test.js +0 -224
  143. package/__tests__/fixtures/entity-resolver-200-mock.json +0 -96
  144. package/__tests__/ids.test.js +0 -45
  145. package/__tests__/integration/ai-chat-history-registry.test.js +0 -228
  146. package/__tests__/integration/aichat-wizard-end-to-end.test.js +0 -282
  147. package/__tests__/integration/cross-adapter-pipelines.test.js +0 -396
  148. package/__tests__/integration/local-data-adapters-pipeline.test.js +0 -373
  149. package/__tests__/integration/social-bilibili-pipeline.test.js +0 -261
  150. package/__tests__/integration/wechat-bootstrap-end-to-end.test.js +0 -390
  151. package/__tests__/key-providers.test.js +0 -126
  152. package/__tests__/kg-derive.test.js +0 -219
  153. package/__tests__/llm-client.test.js +0 -122
  154. package/__tests__/longtail-adapters.test.js +0 -281
  155. package/__tests__/messaging-qq-snapshot.test.js +0 -294
  156. package/__tests__/mobile-extractor-encrypted.test.js +0 -460
  157. package/__tests__/mobile-extractor.test.js +0 -288
  158. package/__tests__/mock-adapter.test.js +0 -93
  159. package/__tests__/prompt-builder.test.js +0 -249
  160. package/__tests__/query-parser.test.js +0 -302
  161. package/__tests__/rag-derive.test.js +0 -169
  162. package/__tests__/registry-readiness.test.js +0 -292
  163. package/__tests__/registry.test.js +0 -420
  164. package/__tests__/salvage-ingest.test.js +0 -97
  165. package/__tests__/schemas.test.js +0 -331
  166. package/__tests__/shopping-adapters.test.js +0 -392
  167. package/__tests__/shopping-eleme-snapshot.test.js +0 -454
  168. package/__tests__/shopping-pinduoduo-snapshot.test.js +0 -484
  169. package/__tests__/shopping-snapshot.test.js +0 -438
  170. package/__tests__/shopping-vipshop-snapshot.test.js +0 -425
  171. package/__tests__/shopping-xianyu-snapshot.test.js +0 -451
  172. package/__tests__/sidecar-contacts-cross-validate.test.js +0 -186
  173. package/__tests__/sidecar-supervisor.test.js +0 -128
  174. package/__tests__/sign-providers.test.js +0 -62
  175. package/__tests__/social-adapters.test.js +0 -280
  176. package/__tests__/social-bilibili-snapshot.test.js +0 -278
  177. package/__tests__/social-douban-snapshot.test.js +0 -351
  178. package/__tests__/social-douyin-im-direct-read.test.js +0 -377
  179. package/__tests__/social-douyin-salvage-collector.test.js +0 -98
  180. package/__tests__/social-douyin-salvage-mapper.test.js +0 -90
  181. package/__tests__/social-douyin-snapshot.test.js +0 -256
  182. package/__tests__/social-kuaishou-snapshot.test.js +0 -362
  183. package/__tests__/social-toutiao-snapshot.test.js +0 -366
  184. package/__tests__/social-weibo-snapshot.test.js +0 -234
  185. package/__tests__/social-weibo-sqlite-device.test.js +0 -174
  186. package/__tests__/social-xiaohongshu-snapshot.test.js +0 -232
  187. package/__tests__/sqlite-leaf-salvage.test.js +0 -97
  188. package/__tests__/travel-adapters.test.js +0 -483
  189. package/__tests__/travel-maps-snapshot.test.js +0 -426
  190. package/__tests__/vault-driver-error.test.js +0 -74
  191. package/__tests__/vault-search-helpers.test.js +0 -104
  192. package/__tests__/vault-search.test.js +0 -423
  193. package/__tests__/vault.test.js +0 -767
  194. package/__tests__/wechat-adapter.test.js +0 -594
  195. package/__tests__/whatsapp-adapter.test.js +0 -138
  196. package/scripts/_make-fixture-all.js +0 -126
  197. package/scripts/_make-fixture-contacts.js +0 -84
  198. package/scripts/evaluate-entity-resolver.js +0 -213
  199. package/scripts/run-native-tests-sandbox.sh +0 -55
  200. package/scripts/smoke-phase-5-5.js +0 -196
  201. package/scripts/smoke-phase-5-7.js +0 -181
  202. package/scripts/smoke-system-data-contacts.js +0 -309
  203. package/scripts/smoke-system-data.js +0 -312
  204. package/vitest.config.js +0 -88
@@ -1,411 +0,0 @@
1
- "use strict";
2
-
3
- import { describe, it, expect, vi } from "vitest";
4
-
5
- const {
6
- EntityResolverEmbeddingStage,
7
- entityResolverCosineSimilarity: cosineSimilarity,
8
- EntityResolverLLMStage,
9
- parseEntityResolverLLMResponse: parseLLMResponse,
10
- EntityResolverWorker,
11
- EntityResolver,
12
- } = require("../lib/entity-resolver");
13
- const { LocalVault } = require("../lib/vault");
14
- const { generateKeyHex } = require("../lib/key-providers");
15
-
16
- const path = require("node:path");
17
- const fs = require("node:fs");
18
- const os = require("node:os");
19
-
20
- // ─── cosineSimilarity ────────────────────────────────────────────────────
21
-
22
- describe("cosineSimilarity", () => {
23
- it("identical vectors → 1", () => {
24
- expect(cosineSimilarity([1, 2, 3], [1, 2, 3])).toBeCloseTo(1, 5);
25
- });
26
-
27
- it("orthogonal vectors → 0", () => {
28
- expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0, 5);
29
- });
30
-
31
- it("opposite vectors → 0 (clamped)", () => {
32
- // Mathematically -1, but we clamp to [0,1]
33
- expect(cosineSimilarity([1, 0], [-1, 0])).toBe(0);
34
- });
35
-
36
- it("empty inputs → 0", () => {
37
- expect(cosineSimilarity([], [])).toBe(0);
38
- expect(cosineSimilarity(null, [1, 2])).toBe(0);
39
- });
40
-
41
- it("zero-norm vectors → 0", () => {
42
- expect(cosineSimilarity([0, 0, 0], [1, 2, 3])).toBe(0);
43
- });
44
-
45
- it("Float32Array works equivalently to plain array", () => {
46
- const a = new Float32Array([1, 2, 3]);
47
- const b = [1, 2, 3];
48
- expect(cosineSimilarity(a, b)).toBeCloseTo(1, 5);
49
- });
50
- });
51
-
52
- // ─── EmbeddingStage with injected embedFn ────────────────────────────────
53
-
54
- describe("EntityResolverEmbeddingStage", () => {
55
- const personA = {
56
- id: "p-a",
57
- type: "person",
58
- names: ["张三", "Zhang"],
59
- identifiers: { email: ["a@x.com"], phone: ["13800001111"] },
60
- };
61
- const personB = {
62
- id: "p-b",
63
- type: "person",
64
- names: ["张三"],
65
- identifiers: { phone: ["13800001111"] },
66
- };
67
-
68
- it("constructor accepts empty opts (defaults Ollama)", () => {
69
- const stage = new EntityResolverEmbeddingStage({});
70
- expect(stage._ollamaUrl).toContain("11434");
71
- expect(stage._model).toBe("nomic-embed-text");
72
- });
73
-
74
- it("constructor rejects non-object opts", () => {
75
- expect(() => new EntityResolverEmbeddingStage(null)).toThrow();
76
- expect(() => new EntityResolverEmbeddingStage("string")).toThrow();
77
- });
78
-
79
- it("compare returns sim + profile", async () => {
80
- const stage = new EntityResolverEmbeddingStage({
81
- embedFn: async (text) => {
82
- // Deterministic stub: hash → vec
83
- return text.length === text.length ? [1, 2, 3] : [3, 2, 1];
84
- },
85
- });
86
- const r = await stage.compare(personA, personB);
87
- expect(r.sim).toBeCloseTo(1, 5);
88
- expect(r.profileA).toContain("张三");
89
- expect(r.profileB).toContain("张三");
90
- });
91
-
92
- it("buildProfile includes name/aliases/identifiers", () => {
93
- const stage = new EntityResolverEmbeddingStage({ embedFn: async () => [1] });
94
- const p = stage.buildProfile(personA);
95
- expect(p).toContain("张三");
96
- expect(p).toContain("Zhang");
97
- expect(p).toContain("email:a@x.com");
98
- expect(p).toContain("phone:13800001111");
99
- });
100
-
101
- it("caches embeddings per person id (1 embedFn call for repeated person)", async () => {
102
- let callCount = 0;
103
- const stage = new EntityResolverEmbeddingStage({
104
- embedFn: async () => { callCount += 1; return [1, 2, 3]; },
105
- });
106
- await stage.compare(personA, personB);
107
- await stage.compare(personA, personB);
108
- expect(callCount).toBe(2); // first compare embeds both; second uses cache
109
- });
110
-
111
- it("cache evicts FIFO when over cacheMaxSize", async () => {
112
- const stage = new EntityResolverEmbeddingStage({
113
- embedFn: async () => [1, 2, 3],
114
- cacheMaxSize: 2,
115
- });
116
- await stage.compare({ id: "p-1", names: ["a"], identifiers: {} }, { id: "p-2", names: ["b"], identifiers: {} });
117
- await stage.compare({ id: "p-3", names: ["c"], identifiers: {} }, { id: "p-4", names: ["d"], identifiers: {} });
118
- expect(stage._cache.size).toBeLessThanOrEqual(2);
119
- });
120
-
121
- it("asStageFn returns bound function", async () => {
122
- const stage = new EntityResolverEmbeddingStage({
123
- embedFn: async () => [1, 0, 0],
124
- });
125
- const fn = stage.asStageFn();
126
- const r = await fn(personA, personB);
127
- expect(r.sim).toBeDefined();
128
- });
129
-
130
- it("rejects non-array embedFn output", async () => {
131
- const stage = new EntityResolverEmbeddingStage({
132
- embedFn: async () => "not an array",
133
- });
134
- await expect(stage.compare(personA, personB)).rejects.toThrow(/Array/);
135
- });
136
- });
137
-
138
- // ─── parseLLMResponse ────────────────────────────────────────────────────
139
-
140
- describe("parseLLMResponse", () => {
141
- it("parses strict JSON", () => {
142
- const r = parseLLMResponse('{"same":true,"confidence":0.9,"reason":"phone match"}');
143
- expect(r.same).toBe(true);
144
- expect(r.confidence).toBe(0.9);
145
- });
146
-
147
- it("parses fenced JSON", () => {
148
- const r = parseLLMResponse('```json\n{"same":false,"confidence":0.8,"reason":"different ids"}\n```');
149
- expect(r.same).toBe(false);
150
- });
151
-
152
- it("parses fenced JSON without language tag", () => {
153
- const r = parseLLMResponse('```\n{"same":null,"confidence":0.5,"reason":"unclear"}\n```');
154
- expect(r.same).toBeNull();
155
- });
156
-
157
- it("falls back to regex extraction from messy preamble", () => {
158
- const text = 'Here is my analysis:\n{"same":true,"confidence":0.85,"reason":"shared phone"}\nLet me know if you have questions.';
159
- const r = parseLLMResponse(text);
160
- expect(r.same).toBe(true);
161
- });
162
-
163
- it("returns null on totally garbage input", () => {
164
- expect(parseLLMResponse("just some prose, no json")).toBeNull();
165
- expect(parseLLMResponse("")).toBeNull();
166
- expect(parseLLMResponse(null)).toBeNull();
167
- });
168
-
169
- it("ignores JSON objects without a 'same' field", () => {
170
- expect(parseLLMResponse('{"foo":"bar"}')).toBeNull();
171
- });
172
- });
173
-
174
- // ─── LLMStage ────────────────────────────────────────────────────────────
175
-
176
- describe("EntityResolverLLMStage", () => {
177
- const a = { id: "p-a", names: ["张三"], identifiers: { phone: ["13800001111"] }, source: { adapter: "email" } };
178
- const b = { id: "p-b", names: ["张三"], identifiers: { phone: ["13800001111"] }, source: { adapter: "alipay" } };
179
-
180
- it("constructor requires llm with .chat()", () => {
181
- expect(() => new EntityResolverLLMStage()).toThrow();
182
- expect(() => new EntityResolverLLMStage({})).toThrow();
183
- expect(() => new EntityResolverLLMStage({ llm: {} })).toThrow();
184
- });
185
-
186
- it("arbitrate maps same:true → verdict yes", async () => {
187
- const stage = new EntityResolverLLMStage({
188
- llm: {
189
- isLocal: true,
190
- chat: async () => ({ text: '{"same":true,"confidence":0.92,"reason":"phone match"}' }),
191
- },
192
- });
193
- const r = await stage.arbitrate(a, b);
194
- expect(r.verdict).toBe("yes");
195
- expect(r.confidence).toBe(0.92);
196
- });
197
-
198
- it("arbitrate maps same:false → verdict no", async () => {
199
- const stage = new EntityResolverLLMStage({
200
- llm: {
201
- isLocal: true,
202
- chat: async () => ({ text: '{"same":false,"confidence":0.8,"reason":"different ids"}' }),
203
- },
204
- });
205
- const r = await stage.arbitrate(a, b);
206
- expect(r.verdict).toBe("no");
207
- });
208
-
209
- it("arbitrate maps same:null → verdict maybe", async () => {
210
- const stage = new EntityResolverLLMStage({
211
- llm: {
212
- isLocal: true,
213
- chat: async () => ({ text: '{"same":null,"confidence":0.5,"reason":"insufficient evidence"}' }),
214
- },
215
- });
216
- const r = await stage.arbitrate(a, b);
217
- expect(r.verdict).toBe("maybe");
218
- });
219
-
220
- it("unparseable LLM response → maybe + reason", async () => {
221
- const stage = new EntityResolverLLMStage({
222
- llm: {
223
- isLocal: true,
224
- chat: async () => ({ text: 'just some prose' }),
225
- },
226
- });
227
- const r = await stage.arbitrate(a, b);
228
- expect(r.verdict).toBe("maybe");
229
- expect(r.confidence).toBe(0);
230
- expect(r.reason).toMatch(/not parseable/);
231
- });
232
-
233
- it("non-local LLM without acceptNonLocal → refuses + returns maybe", async () => {
234
- const stage = new EntityResolverLLMStage({
235
- llm: {
236
- isLocal: false,
237
- chat: async () => { throw new Error("should not be called"); },
238
- },
239
- });
240
- const r = await stage.arbitrate(a, b);
241
- expect(r.verdict).toBe("maybe");
242
- expect(r.reason).toMatch(/non-local/);
243
- });
244
-
245
- it("non-local LLM with acceptNonLocal:true → allowed", async () => {
246
- const stage = new EntityResolverLLMStage({
247
- llm: {
248
- isLocal: false,
249
- chat: async () => ({ text: '{"same":true,"confidence":0.9,"reason":"ok"}' }),
250
- },
251
- acceptNonLocal: true,
252
- });
253
- const r = await stage.arbitrate(a, b);
254
- expect(r.verdict).toBe("yes");
255
- });
256
-
257
- it("LLM throwing propagates (caller handles retry)", async () => {
258
- const stage = new EntityResolverLLMStage({
259
- llm: {
260
- isLocal: true,
261
- chat: async () => { throw new Error("ollama down"); },
262
- },
263
- });
264
- await expect(stage.arbitrate(a, b)).rejects.toThrow(/ollama/);
265
- });
266
-
267
- it("confidence outside [0,1] gets clamped", async () => {
268
- const stage = new EntityResolverLLMStage({
269
- llm: {
270
- isLocal: true,
271
- chat: async () => ({ text: '{"same":true,"confidence":42}' }),
272
- },
273
- });
274
- const r = await stage.arbitrate(a, b);
275
- expect(r.confidence).toBe(1);
276
- });
277
- });
278
-
279
- // ─── EntityResolverWorker ───────────────────────────────────────────────
280
-
281
- function makeMockResolver(initialQueue, drainResults) {
282
- let i = 0;
283
- const queue = [...initialQueue];
284
- return {
285
- drain: async ({ limit }) => {
286
- if (queue.length === 0) {
287
- return { processed: 0, same: 0, different: 0, review: 0, error: 0, skipped: 0 };
288
- }
289
- const result = drainResults[i] || {
290
- processed: Math.min(limit, queue.length),
291
- same: 0, different: 0, review: 0, error: 0, skipped: 0,
292
- };
293
- queue.splice(0, result.processed);
294
- i += 1;
295
- return result;
296
- },
297
- _queueLength: () => queue.length,
298
- };
299
- }
300
-
301
- describe("EntityResolverWorker", () => {
302
- it("constructor requires resolver", () => {
303
- expect(() => new EntityResolverWorker()).toThrow();
304
- expect(() => new EntityResolverWorker({})).toThrow(/resolver/);
305
- });
306
-
307
- it("tick processes one batch and updates stats", async () => {
308
- const resolver = makeMockResolver(["p1", "p2", "p3"], [
309
- { processed: 3, same: 1, different: 1, review: 1, error: 0, skipped: 0 },
310
- ]);
311
- const worker = new EntityResolverWorker({ resolver });
312
- const r = await worker.tick();
313
- expect(r.processed).toBe(3);
314
- const s = worker.stats();
315
- expect(s.batchesProcessed).toBe(1);
316
- expect(s.itemsProcessed).toBe(3);
317
- expect(s.same).toBe(1);
318
- expect(s.review).toBe(1);
319
- });
320
-
321
- it("start + stop cycle works without leaks", async () => {
322
- const resolver = makeMockResolver([], []);
323
- const worker = new EntityResolverWorker({ resolver, idleIntervalMs: 100 });
324
- expect(worker.isRunning()).toBe(false);
325
- worker.start();
326
- expect(worker.isRunning()).toBe(true);
327
- // Give the loop one tick
328
- await new Promise((resolve) => setTimeout(resolve, 250));
329
- await worker.stop();
330
- expect(worker.isRunning()).toBe(false);
331
- });
332
-
333
- it("processes queue then sleeps; onProgress fires per batch", async () => {
334
- const resolver = makeMockResolver(["p1", "p2"], [
335
- { processed: 2, same: 2, different: 0, review: 0, error: 0, skipped: 0 },
336
- ]);
337
- const events = [];
338
- const worker = new EntityResolverWorker({
339
- resolver,
340
- idleIntervalMs: 5000, // long so we observe just the first batch
341
- batchSpacingMs: 1,
342
- onProgress: (e) => events.push(e),
343
- });
344
- worker.start();
345
- // Give time for one batch + post-batch tiny delay
346
- await new Promise((resolve) => setTimeout(resolve, 200));
347
- await worker.stop();
348
- expect(events.length).toBeGreaterThanOrEqual(1);
349
- expect(events[0].batch.same).toBe(2);
350
- });
351
-
352
- it("onProgress listener errors do NOT break the loop", async () => {
353
- const resolver = makeMockResolver(["p1"], [
354
- { processed: 1, same: 0, different: 0, review: 0, error: 0, skipped: 1 },
355
- ]);
356
- const worker = new EntityResolverWorker({
357
- resolver,
358
- idleIntervalMs: 5000,
359
- onProgress: () => { throw new Error("listener boom"); },
360
- });
361
- worker.start();
362
- await new Promise((resolve) => setTimeout(resolve, 200));
363
- await worker.stop();
364
- expect(worker.stats().batchesProcessed).toBeGreaterThanOrEqual(1);
365
- });
366
- });
367
-
368
- // ─── End-to-end: real EntityResolver + real EmbeddingStage stub + real Worker ──
369
-
370
- describe("EntityResolver + EmbeddingStage + LLMStage + Worker integration", () => {
371
- it("queued uncertain pair → embedding high sim → auto-merged", async () => {
372
- const dir = fs.mkdtempSync(path.join(os.tmpdir(), "er-int-"));
373
- const vault = new LocalVault({ path: path.join(dir, "v.db"), key: generateKeyHex() });
374
- vault.open();
375
- try {
376
- // 2 persons sharing only name (rule → uncertain)
377
- const baseSource = { adapter: "test", adapterVersion: "0.1", originalId: "x", capturedAt: Date.now(), capturedBy: "api" };
378
- vault.putPerson({
379
- id: "p-a", type: "person", subtype: "contact",
380
- names: ["张三"], identifiers: {}, ingestedAt: Date.now(),
381
- source: { ...baseSource, adapter: "email", originalId: "1" },
382
- });
383
- vault.putPerson({
384
- id: "p-b", type: "person", subtype: "contact",
385
- names: ["张三"], identifiers: {}, ingestedAt: Date.now(),
386
- source: { ...baseSource, adapter: "alipay", originalId: "2" },
387
- });
388
-
389
- const embedStage = new EntityResolverEmbeddingStage({
390
- embedFn: async () => [1, 0, 0], // deterministic identical vec → sim=1
391
- });
392
- const llmStage = new EntityResolverLLMStage({
393
- llm: { isLocal: true, chat: async () => ({ text: '{"same":true,"confidence":0.9,"reason":"name"}' }) },
394
- });
395
- const resolver = new EntityResolver({
396
- vault,
397
- embeddingStage: embedStage.asStageFn(),
398
- llmStage: llmStage.asStageFn(),
399
- });
400
-
401
- vault.enqueueResolve("p-b");
402
- const worker = new EntityResolverWorker({ resolver, batchSize: 10 });
403
- const r = await worker.tick();
404
- expect(r.same).toBe(1);
405
- expect(vault.getMergeGroupMembers("p-a").sort()).toEqual(["p-a", "p-b"]);
406
- } finally {
407
- try { vault.close(); } catch (_e) {}
408
- try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) {}
409
- }
410
- });
411
- });
@@ -1,249 +0,0 @@
1
- "use strict";
2
-
3
- import { describe, it, expect, beforeEach, afterEach } from "vitest";
4
-
5
- const path = require("node:path");
6
- const fs = require("node:fs");
7
- const os = require("node:os");
8
- const { LocalVault } = require("../lib/vault");
9
- const { generateKeyHex } = require("../lib/key-providers");
10
- const { TARGET_VERSION } = require("../lib/migrations");
11
-
12
- // Helper to spin up a fresh vault each test
13
- function makeVault() {
14
- const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hub-er-test-"));
15
- const dbPath = path.join(dir, "vault.db");
16
- const key = generateKeyHex();
17
- const vault = new LocalVault({ path: dbPath, key });
18
- vault.open();
19
- return { vault, dir };
20
- }
21
-
22
- function cleanup(vault, dir) {
23
- try { vault.close(); } catch (_e) {}
24
- try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) {}
25
- }
26
-
27
- // ─── Migration v2 — Phase 8 tables exist ─────────────────────────────────
28
-
29
- describe("Phase 8 migration v2 — EntityResolver tables", () => {
30
- let vault, dir;
31
- beforeEach(() => { ({ vault, dir } = makeVault()); });
32
- afterEach(() => cleanup(vault, dir));
33
-
34
- it("schemaVersion is current after open()", () => {
35
- expect(vault.schemaVersion()).toBe(TARGET_VERSION);
36
- // Phase 8 ER tables landed in v2; subsequent migrations must not regress.
37
- expect(TARGET_VERSION).toBeGreaterThanOrEqual(2);
38
- });
39
-
40
- it("all 5 new tables exist + are queryable", () => {
41
- const s = vault.stats();
42
- expect(s.mergeGroups).toBe(0);
43
- expect(s.mergeMembers).toBe(0);
44
- expect(s.resolveQueue).toBe(0);
45
- expect(s.reviewQueue).toBe(0);
46
- expect(s.resolveDecisions).toBe(0);
47
- });
48
-
49
- });
50
-
51
- // ─── resolve_queue ───────────────────────────────────────────────────────
52
-
53
- describe("Phase 8 — resolve_queue helpers", () => {
54
- let vault, dir;
55
- beforeEach(() => { ({ vault, dir } = makeVault()); });
56
- afterEach(() => cleanup(vault, dir));
57
-
58
- it("enqueueResolve creates a pending row", () => {
59
- const id = vault.enqueueResolve("person-a");
60
- expect(typeof id === "number" || typeof id === "bigint").toBe(true);
61
- expect(vault.resolveQueueStats().pending).toBe(1);
62
- });
63
-
64
- it("enqueueResolve is idempotent for pending rows", () => {
65
- const a = vault.enqueueResolve("person-a");
66
- const b = vault.enqueueResolve("person-a");
67
- expect(a).toBe(b);
68
- expect(vault.resolveQueueStats().pending).toBe(1);
69
- });
70
-
71
- it("claimResolveBatch marks pending → in-progress + increments attempts", () => {
72
- vault.enqueueResolve("person-a");
73
- vault.enqueueResolve("person-b");
74
- const batch = vault.claimResolveBatch(50);
75
- expect(batch).toHaveLength(2);
76
- expect(batch[0].person_id).toBe("person-a");
77
- expect(batch[0].attempts).toBe(0); // before increment
78
- const stats = vault.resolveQueueStats();
79
- expect(stats.pending).toBe(0);
80
- expect(stats["in-progress"]).toBe(2);
81
- });
82
-
83
- it("completeResolve flips status to done", () => {
84
- vault.enqueueResolve("person-a");
85
- const [row] = vault.claimResolveBatch(10);
86
- vault.completeResolve(row.id);
87
- expect(vault.resolveQueueStats().done).toBe(1);
88
- });
89
-
90
- it("errorResolve re-pends until 3 attempts then marks error", () => {
91
- vault.enqueueResolve("person-a");
92
- let row = vault.claimResolveBatch(10)[0];
93
- vault.errorResolve(row.id, "fail 1");
94
- expect(vault.resolveQueueStats().pending).toBe(1);
95
- row = vault.claimResolveBatch(10)[0];
96
- vault.errorResolve(row.id, "fail 2");
97
- expect(vault.resolveQueueStats().pending).toBe(1);
98
- row = vault.claimResolveBatch(10)[0];
99
- vault.errorResolve(row.id, "fail 3");
100
- expect(vault.resolveQueueStats().error).toBe(1);
101
- expect(vault.resolveQueueStats().pending).toBe(0);
102
- });
103
- });
104
-
105
- // ─── resolve_decisions ───────────────────────────────────────────────────
106
-
107
- describe("Phase 8 — resolve_decisions", () => {
108
- let vault, dir;
109
- beforeEach(() => { ({ vault, dir } = makeVault()); });
110
- afterEach(() => cleanup(vault, dir));
111
-
112
- it("recordResolveDecision lex-orders ids", () => {
113
- vault.recordResolveDecision({
114
- aId: "z-person", bId: "a-person",
115
- verdict: "same", confidence: 0.9, decidedBy: "rule", reason: "test",
116
- });
117
- // Looking up either order returns the same row
118
- const r1 = vault.getResolveDecision("a-person", "z-person");
119
- const r2 = vault.getResolveDecision("z-person", "a-person");
120
- expect(r1).toBeDefined();
121
- expect(r2).toBeDefined();
122
- expect(r1.a_person_id).toBe("a-person");
123
- expect(r1.b_person_id).toBe("z-person");
124
- expect(r1.verdict).toBe("same");
125
- });
126
-
127
- it("recordResolveDecision upserts on conflict", () => {
128
- vault.recordResolveDecision({ aId: "p-a", bId: "p-b", verdict: "same", confidence: 0.7, decidedBy: "embedding" });
129
- vault.recordResolveDecision({ aId: "p-a", bId: "p-b", verdict: "different", confidence: 0.95, decidedBy: "user" });
130
- const r = vault.getResolveDecision("p-a", "p-b");
131
- expect(r.verdict).toBe("different");
132
- expect(r.decided_by).toBe("user");
133
- expect(r.confidence).toBe(0.95);
134
- });
135
- });
136
-
137
- // ─── mergePair / unmergePerson / merge group closure ─────────────────────
138
-
139
- describe("Phase 8 — merge_groups + merge_members", () => {
140
- let vault, dir;
141
- beforeEach(() => { ({ vault, dir } = makeVault()); });
142
- afterEach(() => cleanup(vault, dir));
143
-
144
- it("mergePair creates a new group with 2 members", () => {
145
- const groupId = vault.mergePair({ aId: "p-a", bId: "p-b" });
146
- expect(typeof groupId).toBe("string");
147
- const members = vault.getMergeGroupMembers("p-a");
148
- expect(members.sort()).toEqual(["p-a", "p-b"]);
149
- });
150
-
151
- it("mergePair adds to existing group when one side already in it", () => {
152
- vault.mergePair({ aId: "p-a", bId: "p-b" });
153
- vault.mergePair({ aId: "p-b", bId: "p-c" });
154
- const members = vault.getMergeGroupMembers("p-a");
155
- expect(members.sort()).toEqual(["p-a", "p-b", "p-c"]);
156
- });
157
-
158
- it("mergePair fuses two existing groups", () => {
159
- vault.mergePair({ aId: "p-a", bId: "p-b" });
160
- vault.mergePair({ aId: "p-c", bId: "p-d" });
161
- // a-b group and c-d group exist independently
162
- expect(vault.getMergeGroupMembers("p-a").sort()).toEqual(["p-a", "p-b"]);
163
- expect(vault.getMergeGroupMembers("p-c").sort()).toEqual(["p-c", "p-d"]);
164
- // Fuse them
165
- vault.mergePair({ aId: "p-b", bId: "p-c" });
166
- const all = vault.getMergeGroupMembers("p-a").sort();
167
- expect(all).toEqual(["p-a", "p-b", "p-c", "p-d"]);
168
- // Verify there's only ONE group remaining
169
- const s = vault.stats();
170
- expect(s.mergeGroups).toBe(1);
171
- expect(s.mergeMembers).toBe(4);
172
- });
173
-
174
- it("mergePair on already-same-group is a no-op", () => {
175
- vault.mergePair({ aId: "p-a", bId: "p-b" });
176
- const groupBefore = vault.getMergeGroupMembers("p-a").length;
177
- vault.mergePair({ aId: "p-a", bId: "p-b" });
178
- const groupAfter = vault.getMergeGroupMembers("p-a").length;
179
- expect(groupAfter).toBe(groupBefore);
180
- });
181
-
182
- it("getMergeGroupMembers returns [personId] for unmerged person", () => {
183
- const members = vault.getMergeGroupMembers("loner-1");
184
- expect(members).toEqual(["loner-1"]);
185
- });
186
-
187
- it("unmergePerson removes from group; group of 2 → group dissolves", () => {
188
- vault.mergePair({ aId: "p-a", bId: "p-b" });
189
- const r = vault.unmergePerson("p-a");
190
- expect(r.ok).toBe(true);
191
- expect(vault.getMergeGroupMembers("p-a")).toEqual(["p-a"]);
192
- expect(vault.getMergeGroupMembers("p-b")).toEqual(["p-b"]);
193
- expect(vault.stats().mergeGroups).toBe(0);
194
- });
195
-
196
- it("unmergePerson shrinks group when 3+ members", () => {
197
- vault.mergePair({ aId: "p-a", bId: "p-b" });
198
- vault.mergePair({ aId: "p-b", bId: "p-c" });
199
- vault.unmergePerson("p-c");
200
- expect(vault.getMergeGroupMembers("p-a").sort()).toEqual(["p-a", "p-b"]);
201
- expect(vault.getMergeGroupMembers("p-c")).toEqual(["p-c"]);
202
- expect(vault.stats().mergeGroups).toBe(1);
203
- });
204
-
205
- it("unmergePerson on non-member returns ok:false", () => {
206
- const r = vault.unmergePerson("never-merged");
207
- expect(r.ok).toBe(false);
208
- });
209
- });
210
-
211
- // ─── review_queue ────────────────────────────────────────────────────────
212
-
213
- describe("Phase 8 — review_queue", () => {
214
- let vault, dir;
215
- beforeEach(() => { ({ vault, dir } = makeVault()); });
216
- afterEach(() => cleanup(vault, dir));
217
-
218
- it("enqueueReview lex-orders + returns id", () => {
219
- const id = vault.enqueueReview({
220
- aId: "z-p", bId: "a-p",
221
- embedSim: 0.72, llmVerdict: "maybe", llmReason: "names similar",
222
- llmConfidence: 0.5,
223
- });
224
- expect(id).toBeDefined();
225
- const queue = vault.listReviewQueue();
226
- expect(queue).toHaveLength(1);
227
- expect(queue[0].a_person_id).toBe("a-p");
228
- expect(queue[0].b_person_id).toBe("z-p");
229
- expect(queue[0].embed_sim).toBe(0.72);
230
- });
231
-
232
- it("listReviewQueue only returns pending rows (reviewed_at IS NULL)", () => {
233
- const id1 = vault.enqueueReview({ aId: "p1", bId: "p2", embedSim: 0.7 });
234
- const id2 = vault.enqueueReview({ aId: "p3", bId: "p4", embedSim: 0.75 });
235
- vault.recordReviewDecision({ reviewId: id1, decision: "same" });
236
- const pending = vault.listReviewQueue();
237
- expect(pending).toHaveLength(1);
238
- expect(pending[0].id).toBe(id2);
239
- });
240
-
241
- it("recordReviewDecision rejects invalid decision", () => {
242
- const id = vault.enqueueReview({ aId: "p1", bId: "p2" });
243
- expect(() => vault.recordReviewDecision({ reviewId: id, decision: "garbage" })).toThrow();
244
- });
245
-
246
- it("recordReviewDecision rejects non-existent review id", () => {
247
- expect(() => vault.recordReviewDecision({ reviewId: 9999, decision: "same" })).toThrow();
248
- });
249
- });