hippo-memory 1.44.0 → 1.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (528) hide show
  1. package/README.md +12 -10
  2. package/bin/hippo.js +3 -1
  3. package/dist/api.d.ts +21 -21
  4. package/dist/api.js +69 -42
  5. package/dist/audit.d.ts +1 -1
  6. package/dist/audit.js +9 -0
  7. package/dist/autolearn.d.ts +1 -1
  8. package/dist/autolearn.js +2 -1
  9. package/dist/cli.d.ts +3 -0
  10. package/dist/cli.js +151 -72
  11. package/dist/client.d.ts +4 -21
  12. package/dist/client.js +3 -112
  13. package/dist/config.js +2 -1
  14. package/dist/consolidate.d.ts +3 -0
  15. package/dist/consolidate.js +44 -42
  16. package/dist/dag.d.ts +1 -0
  17. package/dist/dag.js +10 -5
  18. package/dist/dashboard.js +7 -3
  19. package/dist/db.js +12 -23
  20. package/dist/dedupe.d.ts +1 -0
  21. package/dist/dedupe.js +4 -7
  22. package/dist/embedding-provider.js +2 -1
  23. package/dist/embeddings.js +81 -8
  24. package/dist/extract.d.ts +2 -0
  25. package/dist/extract.js +9 -4
  26. package/dist/mcp/server.js +21 -36
  27. package/dist/memory.d.ts +3 -0
  28. package/dist/memory.js +7 -1
  29. package/dist/refine-llm.js +3 -2
  30. package/dist/rerankers/jev.js +3 -2
  31. package/dist/rerankers/llm.js +3 -2
  32. package/dist/search.js +3 -3
  33. package/dist/server.d.ts +5 -0
  34. package/dist/server.js +39 -29
  35. package/dist/shared.js +1 -1
  36. package/dist/store.d.ts +18 -11
  37. package/dist/store.js +144 -63
  38. package/dist/version.d.ts +4 -6
  39. package/dist/version.js +4 -7
  40. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  41. package/extensions/openclaw-plugin/package.json +1 -1
  42. package/openclaw.plugin.json +1 -1
  43. package/package.json +6 -3
  44. package/dist/ablation.d.ts.map +0 -1
  45. package/dist/ablation.js.map +0 -1
  46. package/dist/ambient.d.ts.map +0 -1
  47. package/dist/ambient.js.map +0 -1
  48. package/dist/api.d.ts.map +0 -1
  49. package/dist/api.js.map +0 -1
  50. package/dist/audit-prune.d.ts.map +0 -1
  51. package/dist/audit-prune.js.map +0 -1
  52. package/dist/audit.d.ts.map +0 -1
  53. package/dist/audit.js.map +0 -1
  54. package/dist/auth.d.ts.map +0 -1
  55. package/dist/auth.js.map +0 -1
  56. package/dist/autolearn.d.ts.map +0 -1
  57. package/dist/autolearn.js.map +0 -1
  58. package/dist/availability.d.ts.map +0 -1
  59. package/dist/availability.js.map +0 -1
  60. package/dist/benchmarks/e1.3/incident-recall-eval.js +0 -78
  61. package/dist/benchmarks/e1.3/incident-recall-eval.js.map +0 -1
  62. package/dist/benchmarks/e1.3/slack-1000-event-smoke.js +0 -103
  63. package/dist/benchmarks/e1.3/slack-1000-event-smoke.js.map +0 -1
  64. package/dist/capture.d.ts.map +0 -1
  65. package/dist/capture.js.map +0 -1
  66. package/dist/card-detail.d.ts.map +0 -1
  67. package/dist/card-detail.js.map +0 -1
  68. package/dist/card.d.ts.map +0 -1
  69. package/dist/card.js.map +0 -1
  70. package/dist/cli.d.ts.map +0 -1
  71. package/dist/cli.js.map +0 -1
  72. package/dist/client.d.ts.map +0 -1
  73. package/dist/client.js.map +0 -1
  74. package/dist/compare.d.ts.map +0 -1
  75. package/dist/compare.js.map +0 -1
  76. package/dist/config.d.ts.map +0 -1
  77. package/dist/config.js.map +0 -1
  78. package/dist/connectors/github/backfill.d.ts.map +0 -1
  79. package/dist/connectors/github/backfill.js.map +0 -1
  80. package/dist/connectors/github/cli-impl.d.ts.map +0 -1
  81. package/dist/connectors/github/cli-impl.js.map +0 -1
  82. package/dist/connectors/github/deletion.d.ts.map +0 -1
  83. package/dist/connectors/github/deletion.js.map +0 -1
  84. package/dist/connectors/github/dlq.d.ts.map +0 -1
  85. package/dist/connectors/github/dlq.js.map +0 -1
  86. package/dist/connectors/github/idempotency.d.ts.map +0 -1
  87. package/dist/connectors/github/idempotency.js.map +0 -1
  88. package/dist/connectors/github/ingest.d.ts.map +0 -1
  89. package/dist/connectors/github/ingest.js.map +0 -1
  90. package/dist/connectors/github/octokit-client.d.ts.map +0 -1
  91. package/dist/connectors/github/octokit-client.js.map +0 -1
  92. package/dist/connectors/github/ratelimit.d.ts.map +0 -1
  93. package/dist/connectors/github/ratelimit.js.map +0 -1
  94. package/dist/connectors/github/scope.d.ts.map +0 -1
  95. package/dist/connectors/github/scope.js.map +0 -1
  96. package/dist/connectors/github/signature.d.ts.map +0 -1
  97. package/dist/connectors/github/signature.js.map +0 -1
  98. package/dist/connectors/github/tenant-routing.d.ts.map +0 -1
  99. package/dist/connectors/github/tenant-routing.js.map +0 -1
  100. package/dist/connectors/github/transform.d.ts.map +0 -1
  101. package/dist/connectors/github/transform.js.map +0 -1
  102. package/dist/connectors/github/types.d.ts.map +0 -1
  103. package/dist/connectors/github/types.js.map +0 -1
  104. package/dist/connectors/slack/backfill.d.ts.map +0 -1
  105. package/dist/connectors/slack/backfill.js.map +0 -1
  106. package/dist/connectors/slack/deletion.d.ts.map +0 -1
  107. package/dist/connectors/slack/deletion.js.map +0 -1
  108. package/dist/connectors/slack/dlq.d.ts.map +0 -1
  109. package/dist/connectors/slack/dlq.js.map +0 -1
  110. package/dist/connectors/slack/idempotency.d.ts.map +0 -1
  111. package/dist/connectors/slack/idempotency.js.map +0 -1
  112. package/dist/connectors/slack/ingest.d.ts.map +0 -1
  113. package/dist/connectors/slack/ingest.js.map +0 -1
  114. package/dist/connectors/slack/ratelimit.d.ts.map +0 -1
  115. package/dist/connectors/slack/ratelimit.js.map +0 -1
  116. package/dist/connectors/slack/scope.d.ts.map +0 -1
  117. package/dist/connectors/slack/scope.js.map +0 -1
  118. package/dist/connectors/slack/signature.d.ts.map +0 -1
  119. package/dist/connectors/slack/signature.js.map +0 -1
  120. package/dist/connectors/slack/tenant-routing.d.ts.map +0 -1
  121. package/dist/connectors/slack/tenant-routing.js.map +0 -1
  122. package/dist/connectors/slack/transform.d.ts.map +0 -1
  123. package/dist/connectors/slack/transform.js.map +0 -1
  124. package/dist/connectors/slack/types.d.ts.map +0 -1
  125. package/dist/connectors/slack/types.js.map +0 -1
  126. package/dist/connectors/slack/web-client.d.ts.map +0 -1
  127. package/dist/connectors/slack/web-client.js.map +0 -1
  128. package/dist/connectors/slack/workspaces.d.ts.map +0 -1
  129. package/dist/connectors/slack/workspaces.js.map +0 -1
  130. package/dist/consolidate.d.ts.map +0 -1
  131. package/dist/consolidate.js.map +0 -1
  132. package/dist/correction-latency.d.ts.map +0 -1
  133. package/dist/correction-latency.js.map +0 -1
  134. package/dist/customer-notes.d.ts.map +0 -1
  135. package/dist/customer-notes.js.map +0 -1
  136. package/dist/dag.d.ts.map +0 -1
  137. package/dist/dag.js.map +0 -1
  138. package/dist/dashboard.d.ts.map +0 -1
  139. package/dist/dashboard.js.map +0 -1
  140. package/dist/db.d.ts.map +0 -1
  141. package/dist/db.js.map +0 -1
  142. package/dist/decisions.d.ts.map +0 -1
  143. package/dist/decisions.js.map +0 -1
  144. package/dist/dedupe.d.ts.map +0 -1
  145. package/dist/dedupe.js.map +0 -1
  146. package/dist/embedding-provider.d.ts.map +0 -1
  147. package/dist/embedding-provider.js.map +0 -1
  148. package/dist/embeddings.d.ts.map +0 -1
  149. package/dist/embeddings.js.map +0 -1
  150. package/dist/eval-suite.d.ts.map +0 -1
  151. package/dist/eval-suite.js.map +0 -1
  152. package/dist/eval.d.ts.map +0 -1
  153. package/dist/eval.js.map +0 -1
  154. package/dist/extensions/openclaw-plugin/index.js.map +0 -1
  155. package/dist/extract.d.ts.map +0 -1
  156. package/dist/extract.js.map +0 -1
  157. package/dist/forward-claim-detector.d.ts.map +0 -1
  158. package/dist/forward-claim-detector.js.map +0 -1
  159. package/dist/goals.d.ts.map +0 -1
  160. package/dist/goals.js.map +0 -1
  161. package/dist/graph-extract.d.ts.map +0 -1
  162. package/dist/graph-extract.js.map +0 -1
  163. package/dist/graph-recall.d.ts.map +0 -1
  164. package/dist/graph-recall.js.map +0 -1
  165. package/dist/graph-stream.d.ts.map +0 -1
  166. package/dist/graph-stream.js.map +0 -1
  167. package/dist/graph-view.d.ts.map +0 -1
  168. package/dist/graph-view.js.map +0 -1
  169. package/dist/graph.d.ts.map +0 -1
  170. package/dist/graph.js.map +0 -1
  171. package/dist/handoff.d.ts.map +0 -1
  172. package/dist/handoff.js.map +0 -1
  173. package/dist/hooks.d.ts.map +0 -1
  174. package/dist/hooks.js.map +0 -1
  175. package/dist/importers.d.ts.map +0 -1
  176. package/dist/importers.js.map +0 -1
  177. package/dist/incidents.d.ts.map +0 -1
  178. package/dist/incidents.js.map +0 -1
  179. package/dist/index.d.ts.map +0 -1
  180. package/dist/index.js.map +0 -1
  181. package/dist/invalidation.d.ts.map +0 -1
  182. package/dist/invalidation.js.map +0 -1
  183. package/dist/mcp/framing.d.ts.map +0 -1
  184. package/dist/mcp/framing.js.map +0 -1
  185. package/dist/mcp/server.d.ts.map +0 -1
  186. package/dist/mcp/server.js.map +0 -1
  187. package/dist/memory-value-weights.d.ts.map +0 -1
  188. package/dist/memory-value-weights.js.map +0 -1
  189. package/dist/memory-value.d.ts.map +0 -1
  190. package/dist/memory-value.js.map +0 -1
  191. package/dist/memory.d.ts.map +0 -1
  192. package/dist/memory.js.map +0 -1
  193. package/dist/multihop.d.ts.map +0 -1
  194. package/dist/multihop.js.map +0 -1
  195. package/dist/owner-validation.d.ts.map +0 -1
  196. package/dist/owner-validation.js.map +0 -1
  197. package/dist/path-context.d.ts.map +0 -1
  198. package/dist/path-context.js.map +0 -1
  199. package/dist/physics-config.d.ts.map +0 -1
  200. package/dist/physics-config.js.map +0 -1
  201. package/dist/physics-state.d.ts.map +0 -1
  202. package/dist/physics-state.js.map +0 -1
  203. package/dist/physics.d.ts.map +0 -1
  204. package/dist/physics.js.map +0 -1
  205. package/dist/policies.d.ts.map +0 -1
  206. package/dist/policies.js.map +0 -1
  207. package/dist/postinstall.d.ts.map +0 -1
  208. package/dist/postinstall.js.map +0 -1
  209. package/dist/predictions.d.ts.map +0 -1
  210. package/dist/predictions.js.map +0 -1
  211. package/dist/processes.d.ts.map +0 -1
  212. package/dist/processes.js.map +0 -1
  213. package/dist/project-briefs.d.ts.map +0 -1
  214. package/dist/project-briefs.js.map +0 -1
  215. package/dist/project-identity.d.ts.map +0 -1
  216. package/dist/project-identity.js.map +0 -1
  217. package/dist/provenance-coverage.d.ts.map +0 -1
  218. package/dist/provenance-coverage.js.map +0 -1
  219. package/dist/rate-limit.d.ts.map +0 -1
  220. package/dist/rate-limit.js.map +0 -1
  221. package/dist/raw-archive-mirror-cleanup.d.ts.map +0 -1
  222. package/dist/raw-archive-mirror-cleanup.js.map +0 -1
  223. package/dist/raw-archive.d.ts.map +0 -1
  224. package/dist/raw-archive.js.map +0 -1
  225. package/dist/recall-history.d.ts.map +0 -1
  226. package/dist/recall-history.js.map +0 -1
  227. package/dist/recall-scope.d.ts.map +0 -1
  228. package/dist/recall-scope.js.map +0 -1
  229. package/dist/recall-trace.d.ts.map +0 -1
  230. package/dist/recall-trace.js.map +0 -1
  231. package/dist/refine-llm.d.ts.map +0 -1
  232. package/dist/refine-llm.js.map +0 -1
  233. package/dist/reject-flow.d.ts.map +0 -1
  234. package/dist/reject-flow.js.map +0 -1
  235. package/dist/rejection.d.ts.map +0 -1
  236. package/dist/rejection.js.map +0 -1
  237. package/dist/replay.d.ts.map +0 -1
  238. package/dist/replay.js.map +0 -1
  239. package/dist/rerankers/cross-encoder.d.ts.map +0 -1
  240. package/dist/rerankers/cross-encoder.js.map +0 -1
  241. package/dist/rerankers/index.d.ts.map +0 -1
  242. package/dist/rerankers/index.js.map +0 -1
  243. package/dist/rerankers/jev.d.ts.map +0 -1
  244. package/dist/rerankers/jev.js.map +0 -1
  245. package/dist/rerankers/llm.d.ts.map +0 -1
  246. package/dist/rerankers/llm.js.map +0 -1
  247. package/dist/rerankers/types.d.ts.map +0 -1
  248. package/dist/rerankers/types.js.map +0 -1
  249. package/dist/rrf.d.ts.map +0 -1
  250. package/dist/rrf.js.map +0 -1
  251. package/dist/salience.d.ts.map +0 -1
  252. package/dist/salience.js.map +0 -1
  253. package/dist/scheduler.d.ts.map +0 -1
  254. package/dist/scheduler.js.map +0 -1
  255. package/dist/scope.d.ts.map +0 -1
  256. package/dist/scope.js.map +0 -1
  257. package/dist/search.d.ts.map +0 -1
  258. package/dist/search.js.map +0 -1
  259. package/dist/secret-detect.d.ts.map +0 -1
  260. package/dist/secret-detect.js.map +0 -1
  261. package/dist/server-detect.d.ts.map +0 -1
  262. package/dist/server-detect.js.map +0 -1
  263. package/dist/server.d.ts.map +0 -1
  264. package/dist/server.js.map +0 -1
  265. package/dist/shared.d.ts.map +0 -1
  266. package/dist/shared.js.map +0 -1
  267. package/dist/skills.d.ts.map +0 -1
  268. package/dist/skills.js.map +0 -1
  269. package/dist/sleep-redact.d.ts +0 -58
  270. package/dist/sleep-redact.d.ts.map +0 -1
  271. package/dist/sleep-redact.js +0 -80
  272. package/dist/sleep-redact.js.map +0 -1
  273. package/dist/src/ablation.js +0 -138
  274. package/dist/src/ablation.js.map +0 -1
  275. package/dist/src/ambient.js +0 -148
  276. package/dist/src/ambient.js.map +0 -1
  277. package/dist/src/api.js +0 -2109
  278. package/dist/src/api.js.map +0 -1
  279. package/dist/src/audit-prune.js +0 -109
  280. package/dist/src/audit-prune.js.map +0 -1
  281. package/dist/src/audit.js +0 -213
  282. package/dist/src/audit.js.map +0 -1
  283. package/dist/src/auth.js +0 -97
  284. package/dist/src/auth.js.map +0 -1
  285. package/dist/src/autolearn.js +0 -174
  286. package/dist/src/autolearn.js.map +0 -1
  287. package/dist/src/availability.js +0 -94
  288. package/dist/src/availability.js.map +0 -1
  289. package/dist/src/capture.js +0 -1346
  290. package/dist/src/capture.js.map +0 -1
  291. package/dist/src/card-detail.js +0 -15
  292. package/dist/src/card-detail.js.map +0 -1
  293. package/dist/src/card.js +0 -19
  294. package/dist/src/card.js.map +0 -1
  295. package/dist/src/cli.js +0 -9387
  296. package/dist/src/cli.js.map +0 -1
  297. package/dist/src/client.js +0 -237
  298. package/dist/src/client.js.map +0 -1
  299. package/dist/src/compare.js +0 -122
  300. package/dist/src/compare.js.map +0 -1
  301. package/dist/src/config.js +0 -135
  302. package/dist/src/config.js.map +0 -1
  303. package/dist/src/connectors/github/backfill.js +0 -281
  304. package/dist/src/connectors/github/backfill.js.map +0 -1
  305. package/dist/src/connectors/github/cli-impl.js +0 -234
  306. package/dist/src/connectors/github/cli-impl.js.map +0 -1
  307. package/dist/src/connectors/github/deletion.js +0 -85
  308. package/dist/src/connectors/github/deletion.js.map +0 -1
  309. package/dist/src/connectors/github/dlq.js +0 -190
  310. package/dist/src/connectors/github/dlq.js.map +0 -1
  311. package/dist/src/connectors/github/idempotency.js +0 -27
  312. package/dist/src/connectors/github/idempotency.js.map +0 -1
  313. package/dist/src/connectors/github/ingest.js +0 -156
  314. package/dist/src/connectors/github/ingest.js.map +0 -1
  315. package/dist/src/connectors/github/octokit-client.js +0 -70
  316. package/dist/src/connectors/github/octokit-client.js.map +0 -1
  317. package/dist/src/connectors/github/ratelimit.js +0 -31
  318. package/dist/src/connectors/github/ratelimit.js.map +0 -1
  319. package/dist/src/connectors/github/scope.js +0 -13
  320. package/dist/src/connectors/github/scope.js.map +0 -1
  321. package/dist/src/connectors/github/signature.js +0 -81
  322. package/dist/src/connectors/github/signature.js.map +0 -1
  323. package/dist/src/connectors/github/tenant-routing.js +0 -69
  324. package/dist/src/connectors/github/tenant-routing.js.map +0 -1
  325. package/dist/src/connectors/github/transform.js +0 -103
  326. package/dist/src/connectors/github/transform.js.map +0 -1
  327. package/dist/src/connectors/github/types.js +0 -100
  328. package/dist/src/connectors/github/types.js.map +0 -1
  329. package/dist/src/connectors/slack/backfill.js +0 -78
  330. package/dist/src/connectors/slack/backfill.js.map +0 -1
  331. package/dist/src/connectors/slack/deletion.js +0 -59
  332. package/dist/src/connectors/slack/deletion.js.map +0 -1
  333. package/dist/src/connectors/slack/dlq.js +0 -224
  334. package/dist/src/connectors/slack/dlq.js.map +0 -1
  335. package/dist/src/connectors/slack/idempotency.js +0 -31
  336. package/dist/src/connectors/slack/idempotency.js.map +0 -1
  337. package/dist/src/connectors/slack/ingest.js +0 -109
  338. package/dist/src/connectors/slack/ingest.js.map +0 -1
  339. package/dist/src/connectors/slack/ratelimit.js +0 -18
  340. package/dist/src/connectors/slack/ratelimit.js.map +0 -1
  341. package/dist/src/connectors/slack/scope.js +0 -13
  342. package/dist/src/connectors/slack/scope.js.map +0 -1
  343. package/dist/src/connectors/slack/signature.js +0 -27
  344. package/dist/src/connectors/slack/signature.js.map +0 -1
  345. package/dist/src/connectors/slack/tenant-routing.js +0 -41
  346. package/dist/src/connectors/slack/tenant-routing.js.map +0 -1
  347. package/dist/src/connectors/slack/transform.js +0 -47
  348. package/dist/src/connectors/slack/transform.js.map +0 -1
  349. package/dist/src/connectors/slack/types.js +0 -35
  350. package/dist/src/connectors/slack/types.js.map +0 -1
  351. package/dist/src/connectors/slack/web-client.js +0 -43
  352. package/dist/src/connectors/slack/web-client.js.map +0 -1
  353. package/dist/src/connectors/slack/workspaces.js +0 -62
  354. package/dist/src/connectors/slack/workspaces.js.map +0 -1
  355. package/dist/src/consolidate.js +0 -978
  356. package/dist/src/consolidate.js.map +0 -1
  357. package/dist/src/correction-latency.js +0 -74
  358. package/dist/src/correction-latency.js.map +0 -1
  359. package/dist/src/customer-notes.js +0 -325
  360. package/dist/src/customer-notes.js.map +0 -1
  361. package/dist/src/dag.js +0 -352
  362. package/dist/src/dag.js.map +0 -1
  363. package/dist/src/dashboard.js +0 -258
  364. package/dist/src/dashboard.js.map +0 -1
  365. package/dist/src/db.js +0 -2731
  366. package/dist/src/db.js.map +0 -1
  367. package/dist/src/decisions.js +0 -304
  368. package/dist/src/decisions.js.map +0 -1
  369. package/dist/src/dedupe.js +0 -141
  370. package/dist/src/dedupe.js.map +0 -1
  371. package/dist/src/embedding-provider.js +0 -314
  372. package/dist/src/embedding-provider.js.map +0 -1
  373. package/dist/src/embeddings.js +0 -543
  374. package/dist/src/embeddings.js.map +0 -1
  375. package/dist/src/eval-suite.js +0 -294
  376. package/dist/src/eval-suite.js.map +0 -1
  377. package/dist/src/eval.js +0 -187
  378. package/dist/src/eval.js.map +0 -1
  379. package/dist/src/extract.js +0 -117
  380. package/dist/src/extract.js.map +0 -1
  381. package/dist/src/forward-claim-detector.js +0 -117
  382. package/dist/src/forward-claim-detector.js.map +0 -1
  383. package/dist/src/goals.js +0 -399
  384. package/dist/src/goals.js.map +0 -1
  385. package/dist/src/graph-extract.js +0 -314
  386. package/dist/src/graph-extract.js.map +0 -1
  387. package/dist/src/graph-recall.js +0 -277
  388. package/dist/src/graph-recall.js.map +0 -1
  389. package/dist/src/graph-stream.js +0 -176
  390. package/dist/src/graph-stream.js.map +0 -1
  391. package/dist/src/graph-view.js +0 -310
  392. package/dist/src/graph-view.js.map +0 -1
  393. package/dist/src/graph.js +0 -762
  394. package/dist/src/graph.js.map +0 -1
  395. package/dist/src/handoff.js +0 -65
  396. package/dist/src/handoff.js.map +0 -1
  397. package/dist/src/hooks.js +0 -926
  398. package/dist/src/hooks.js.map +0 -1
  399. package/dist/src/importers.js +0 -881
  400. package/dist/src/importers.js.map +0 -1
  401. package/dist/src/incidents.js +0 -336
  402. package/dist/src/incidents.js.map +0 -1
  403. package/dist/src/index.js +0 -32
  404. package/dist/src/index.js.map +0 -1
  405. package/dist/src/invalidation.js +0 -129
  406. package/dist/src/invalidation.js.map +0 -1
  407. package/dist/src/mcp/framing.js +0 -45
  408. package/dist/src/mcp/framing.js.map +0 -1
  409. package/dist/src/mcp/server.js +0 -1298
  410. package/dist/src/mcp/server.js.map +0 -1
  411. package/dist/src/memory-value-weights.js +0 -31
  412. package/dist/src/memory-value-weights.js.map +0 -1
  413. package/dist/src/memory-value.js +0 -261
  414. package/dist/src/memory-value.js.map +0 -1
  415. package/dist/src/memory.js +0 -425
  416. package/dist/src/memory.js.map +0 -1
  417. package/dist/src/multihop.js +0 -35
  418. package/dist/src/multihop.js.map +0 -1
  419. package/dist/src/owner-validation.js +0 -56
  420. package/dist/src/owner-validation.js.map +0 -1
  421. package/dist/src/path-context.js +0 -48
  422. package/dist/src/path-context.js.map +0 -1
  423. package/dist/src/physics-config.js +0 -26
  424. package/dist/src/physics-config.js.map +0 -1
  425. package/dist/src/physics-state.js +0 -172
  426. package/dist/src/physics-state.js.map +0 -1
  427. package/dist/src/physics.js +0 -378
  428. package/dist/src/physics.js.map +0 -1
  429. package/dist/src/policies.js +0 -414
  430. package/dist/src/policies.js.map +0 -1
  431. package/dist/src/postinstall.js +0 -101
  432. package/dist/src/postinstall.js.map +0 -1
  433. package/dist/src/predictions.js +0 -629
  434. package/dist/src/predictions.js.map +0 -1
  435. package/dist/src/processes.js +0 -349
  436. package/dist/src/processes.js.map +0 -1
  437. package/dist/src/project-briefs.js +0 -492
  438. package/dist/src/project-briefs.js.map +0 -1
  439. package/dist/src/project-identity.js +0 -200
  440. package/dist/src/project-identity.js.map +0 -1
  441. package/dist/src/provenance-coverage.js +0 -23
  442. package/dist/src/provenance-coverage.js.map +0 -1
  443. package/dist/src/rate-limit.js +0 -60
  444. package/dist/src/rate-limit.js.map +0 -1
  445. package/dist/src/raw-archive-mirror-cleanup.js +0 -55
  446. package/dist/src/raw-archive-mirror-cleanup.js.map +0 -1
  447. package/dist/src/raw-archive.js +0 -91
  448. package/dist/src/raw-archive.js.map +0 -1
  449. package/dist/src/recall-history.js +0 -235
  450. package/dist/src/recall-history.js.map +0 -1
  451. package/dist/src/recall-scope.js +0 -89
  452. package/dist/src/recall-scope.js.map +0 -1
  453. package/dist/src/recall-trace.js +0 -186
  454. package/dist/src/recall-trace.js.map +0 -1
  455. package/dist/src/refine-llm.js +0 -156
  456. package/dist/src/refine-llm.js.map +0 -1
  457. package/dist/src/reject-flow.js +0 -209
  458. package/dist/src/reject-flow.js.map +0 -1
  459. package/dist/src/rejection.js +0 -160
  460. package/dist/src/rejection.js.map +0 -1
  461. package/dist/src/replay.js +0 -122
  462. package/dist/src/replay.js.map +0 -1
  463. package/dist/src/rerankers/cross-encoder.js +0 -137
  464. package/dist/src/rerankers/cross-encoder.js.map +0 -1
  465. package/dist/src/rerankers/index.js +0 -20
  466. package/dist/src/rerankers/index.js.map +0 -1
  467. package/dist/src/rerankers/jev.js +0 -119
  468. package/dist/src/rerankers/jev.js.map +0 -1
  469. package/dist/src/rerankers/llm.js +0 -80
  470. package/dist/src/rerankers/llm.js.map +0 -1
  471. package/dist/src/rerankers/types.js +0 -2
  472. package/dist/src/rerankers/types.js.map +0 -1
  473. package/dist/src/rrf.js +0 -61
  474. package/dist/src/rrf.js.map +0 -1
  475. package/dist/src/salience.js +0 -74
  476. package/dist/src/salience.js.map +0 -1
  477. package/dist/src/scheduler.js +0 -77
  478. package/dist/src/scheduler.js.map +0 -1
  479. package/dist/src/scope.js +0 -35
  480. package/dist/src/scope.js.map +0 -1
  481. package/dist/src/search.js +0 -1003
  482. package/dist/src/search.js.map +0 -1
  483. package/dist/src/secret-detect.js +0 -95
  484. package/dist/src/secret-detect.js.map +0 -1
  485. package/dist/src/server-detect.js +0 -231
  486. package/dist/src/server-detect.js.map +0 -1
  487. package/dist/src/server.js +0 -3255
  488. package/dist/src/server.js.map +0 -1
  489. package/dist/src/shared.js +0 -502
  490. package/dist/src/shared.js.map +0 -1
  491. package/dist/src/skills.js +0 -346
  492. package/dist/src/skills.js.map +0 -1
  493. package/dist/src/sleep-redact.js +0 -80
  494. package/dist/src/sleep-redact.js.map +0 -1
  495. package/dist/src/sso.js +0 -22
  496. package/dist/src/sso.js.map +0 -1
  497. package/dist/src/stdin.js +0 -41
  498. package/dist/src/stdin.js.map +0 -1
  499. package/dist/src/store.js +0 -3749
  500. package/dist/src/store.js.map +0 -1
  501. package/dist/src/tenant.js +0 -17
  502. package/dist/src/tenant.js.map +0 -1
  503. package/dist/src/trace.js +0 -72
  504. package/dist/src/trace.js.map +0 -1
  505. package/dist/src/version.js +0 -40
  506. package/dist/src/version.js.map +0 -1
  507. package/dist/src/working-memory.js +0 -157
  508. package/dist/src/working-memory.js.map +0 -1
  509. package/dist/src/yaml.js +0 -107
  510. package/dist/src/yaml.js.map +0 -1
  511. package/dist/sso.d.ts +0 -13
  512. package/dist/sso.d.ts.map +0 -1
  513. package/dist/sso.js +0 -22
  514. package/dist/sso.js.map +0 -1
  515. package/dist/stdin.d.ts.map +0 -1
  516. package/dist/stdin.js.map +0 -1
  517. package/dist/store.d.ts.map +0 -1
  518. package/dist/store.js.map +0 -1
  519. package/dist/tenant.d.ts.map +0 -1
  520. package/dist/tenant.js.map +0 -1
  521. package/dist/trace.d.ts.map +0 -1
  522. package/dist/trace.js.map +0 -1
  523. package/dist/version.d.ts.map +0 -1
  524. package/dist/version.js.map +0 -1
  525. package/dist/working-memory.d.ts.map +0 -1
  526. package/dist/working-memory.js.map +0 -1
  527. package/dist/yaml.d.ts.map +0 -1
  528. package/dist/yaml.js.map +0 -1
@@ -1,294 +0,0 @@
1
- /**
2
- * Self-contained feature evaluation suite for hippo.
3
- *
4
- * Creates a synthetic memory corpus with known ground truth, runs searches
5
- * per feature category, and reports per-feature metrics with regression
6
- * detection against a saved baseline.
7
- *
8
- * Design goals:
9
- * - Zero API calls (no LLM judge, no embeddings)
10
- * - Deterministic (fixed timestamps, content, IDs)
11
- * - Fast (<60s for full suite)
12
- * - Per-feature breakdown so you see exactly what a change helped/hurt
13
- */
14
- import { createMemory, Layer } from './memory.js';
15
- import { search } from './search.js';
16
- import { multihopSearch } from './multihop.js';
17
- import { mrr, recallAtK, ndcgAtK } from './eval.js';
18
- // ---------------------------------------------------------------------------
19
- // Synthetic corpus — deterministic, no API calls
20
- // ---------------------------------------------------------------------------
21
- const BASE_DATE = new Date('2026-01-15T10:00:00Z');
22
- function dateOffset(days) {
23
- const d = new Date(BASE_DATE.getTime() + days * 86400000);
24
- return d.toISOString();
25
- }
26
- function mem(id, content, opts = {}) {
27
- const entry = createMemory(content, {
28
- layer: opts.layer ?? Layer.Episodic,
29
- tags: opts.tags ?? [],
30
- baseHalfLifeDays: 30,
31
- extracted_from: opts.extracted_from,
32
- dag_level: opts.dag_level,
33
- dag_parent_id: opts.dag_parent_id,
34
- });
35
- entry.id = id;
36
- if (opts.created)
37
- entry.created = opts.created;
38
- return entry;
39
- }
40
- export function buildSyntheticCorpus() {
41
- const entries = [];
42
- const cases = [];
43
- // =========================================================================
44
- // 1. DIRECT RECALL — basic keyword matching
45
- // =========================================================================
46
- entries.push(mem('dr-1', 'The PostgreSQL database migration failed because the users table had a NOT NULL constraint on the email column', { created: dateOffset(1), tags: ['topic:database'] }), mem('dr-2', 'React component rendering performance improved by 40% after memoizing the expensive computation in useMemo', { created: dateOffset(2), tags: ['topic:frontend'] }), mem('dr-3', 'The API rate limiter should use a sliding window algorithm instead of fixed window to prevent burst traffic', { created: dateOffset(3), tags: ['topic:api'] }), mem('dr-4', 'Docker container memory limits need to be set to 512MB for the worker service to prevent OOM kills', { created: dateOffset(4), tags: ['topic:devops'] }), mem('dr-5', 'The JWT token expiration was set to 24 hours but should be reduced to 1 hour for security compliance', { created: dateOffset(5), tags: ['topic:security'] }), mem('dr-6', 'Webpack bundle size grew to 2.3MB because lodash was imported as a whole instead of cherry-picking', { created: dateOffset(6), tags: ['topic:frontend'] }), mem('dr-7', 'The Redis cache TTL for user sessions should match the JWT expiration to prevent stale sessions', { created: dateOffset(7), tags: ['topic:caching'] }), mem('dr-8', 'GraphQL resolver for nested comments has N+1 query problem solved by DataLoader batching', { created: dateOffset(8), tags: ['topic:api'] }));
47
- cases.push({ id: 'dr-q1', category: 'direct-recall', query: 'PostgreSQL migration NOT NULL constraint', expectedIds: ['dr-1'], description: 'exact keyword match on DB migration' }, { id: 'dr-q2', category: 'direct-recall', query: 'React useMemo performance memoizing', expectedIds: ['dr-2'], description: 'React performance optimization' }, { id: 'dr-q3', category: 'direct-recall', query: 'sliding window rate limiter API', expectedIds: ['dr-3'], description: 'API rate limiting approach' }, { id: 'dr-q4', category: 'direct-recall', query: 'Docker OOM memory limits worker', expectedIds: ['dr-4'], description: 'Docker memory config' }, { id: 'dr-q5', category: 'direct-recall', query: 'JWT token expiration security', expectedIds: ['dr-5'], description: 'JWT security setting' }, { id: 'dr-q6', category: 'direct-recall', query: 'webpack bundle size lodash', expectedIds: ['dr-6'], description: 'bundle size issue' }, { id: 'dr-q7', category: 'direct-recall', query: 'Redis cache TTL session expiration', expectedIds: ['dr-7'], description: 'cache TTL config' }, { id: 'dr-q8', category: 'direct-recall', query: 'GraphQL N+1 DataLoader batching', expectedIds: ['dr-8'], description: 'GraphQL N+1 fix' });
48
- // =========================================================================
49
- // 2. EXTRACTION PREFERENCE — extracted facts should rank above raw source
50
- // =========================================================================
51
- entries.push(mem('ep-src-1', 'speaker:Alice: So we had this big meeting yesterday about the deployment pipeline and Bob mentioned that the staging environment is using Kubernetes 1.28 and we should upgrade to 1.30 before the end of Q2 because of the security patches', {
52
- created: dateOffset(10), tags: ['speaker:Alice', 'topic:infrastructure', 'session:meeting-1'],
53
- }), mem('ep-ext-1', 'The staging environment runs Kubernetes 1.28 and needs to be upgraded to 1.30 before end of Q2 for security patches', {
54
- created: dateOffset(10), layer: Layer.Semantic, tags: ['speaker:Alice', 'topic:infrastructure', 'extracted'],
55
- extracted_from: 'ep-src-1',
56
- }), mem('ep-src-2', 'speaker:Bob: Yeah and the thing about the monitoring is that we switched from Datadog to Grafana last month and the alerting rules still need to be migrated, Carol was supposed to handle that but she has been busy with the frontend rewrite', {
57
- created: dateOffset(11), tags: ['speaker:Bob', 'topic:monitoring', 'session:meeting-1'],
58
- }), mem('ep-ext-2', 'The team switched from Datadog to Grafana last month but alerting rules have not been migrated yet. Carol is responsible but blocked by the frontend rewrite.', {
59
- created: dateOffset(11), layer: Layer.Semantic, tags: ['speaker:Bob', 'topic:monitoring', 'extracted'],
60
- extracted_from: 'ep-src-2',
61
- }), mem('ep-src-3', 'speaker:Carol: The login page redesign is almost done, I just need to wire up the OAuth2 PKCE flow with the new identity provider and write the integration tests', {
62
- created: dateOffset(12), tags: ['speaker:Carol', 'topic:auth', 'session:meeting-2'],
63
- }), mem('ep-ext-3', 'Carol is nearly done with the login page redesign. Remaining work: wire up OAuth2 PKCE flow with the new identity provider and write integration tests.', {
64
- created: dateOffset(12), layer: Layer.Semantic, tags: ['speaker:Carol', 'topic:auth', 'extracted'],
65
- extracted_from: 'ep-src-3',
66
- }));
67
- cases.push({ id: 'ep-q1', category: 'extraction-preference', query: 'Kubernetes upgrade staging environment', expectedIds: ['ep-ext-1'], description: 'extracted fact about K8s upgrade should rank above raw utterance' }, { id: 'ep-q2', category: 'extraction-preference', query: 'Datadog Grafana alerting migration', expectedIds: ['ep-ext-2'], description: 'extracted fact about monitoring switch' }, { id: 'ep-q3', category: 'extraction-preference', query: 'OAuth2 PKCE login page redesign', expectedIds: ['ep-ext-3'], description: 'extracted fact about auth work' });
68
- // =========================================================================
69
- // 3. DAG DRILL-DOWN — summary nodes should surface children
70
- // =========================================================================
71
- entries.push(mem('dag-child-1', 'The API response time for /users endpoint degraded from 50ms to 300ms after adding the permissions check', {
72
- created: dateOffset(15), layer: Layer.Semantic, tags: ['topic:api-performance', 'extracted'],
73
- dag_level: 1,
74
- }), mem('dag-child-2', 'The /orders endpoint latency spiked to 500ms because of a missing index on the orders.created_at column', {
75
- created: dateOffset(15), layer: Layer.Semantic, tags: ['topic:api-performance', 'extracted'],
76
- dag_level: 1,
77
- }), mem('dag-child-3', 'Batch API endpoint /reports/generate takes 8 seconds because it runs synchronously instead of using a job queue', {
78
- created: dateOffset(15), layer: Layer.Semantic, tags: ['topic:api-performance', 'extracted'],
79
- dag_level: 1,
80
- }), mem('dag-summary-1', 'API performance issues: /users degraded to 300ms (permissions check), /orders spiked to 500ms (missing index), /reports takes 8s (needs job queue)', {
81
- created: dateOffset(16), layer: Layer.Semantic, tags: ['topic:api-performance'],
82
- dag_level: 2, dag_parent_id: undefined,
83
- }));
84
- // Link children to parent
85
- for (const child of [entries.find(e => e.id === 'dag-child-1'), entries.find(e => e.id === 'dag-child-2'), entries.find(e => e.id === 'dag-child-3')]) {
86
- child.dag_parent_id = 'dag-summary-1';
87
- }
88
- cases.push({ id: 'dag-q1', category: 'dag-drilldown', query: 'API performance problems latency', expectedIds: ['dag-child-1', 'dag-child-2', 'dag-child-3', 'dag-summary-1'], description: 'summary should drill down to all children' }, { id: 'dag-q2', category: 'dag-drilldown', query: 'endpoint response time degradation', expectedIds: ['dag-child-1', 'dag-child-2', 'dag-summary-1'], description: 'query matching summary should surface relevant children' });
89
- // =========================================================================
90
- // 4. TEMPORAL — recency/oldest cues should affect ranking
91
- // =========================================================================
92
- entries.push(mem('tmp-1', 'The team decided to use TypeScript for the new service', { created: dateOffset(-30), tags: ['topic:architecture'] }), mem('tmp-2', 'The team evaluated Rust as an alternative language for the service', { created: dateOffset(-20), tags: ['topic:architecture'] }), mem('tmp-3', 'The team added Go as a candidate language for the service rewrite', { created: dateOffset(-10), tags: ['topic:architecture'] }), mem('tmp-4', 'The team finalized the language choice as Go for the service rewrite', { created: dateOffset(-1), tags: ['topic:architecture'] }));
93
- cases.push({ id: 'tmp-q1', category: 'temporal', query: 'what did the team recently decide about the service language', expectedIds: ['tmp-4'], description: 'recent cue should boost newest entry' }, { id: 'tmp-q2', category: 'temporal', query: 'what was the first language choice for the service', expectedIds: ['tmp-1'], description: 'oldest cue should boost earliest entry' }, { id: 'tmp-q3', category: 'temporal', query: 'latest update on the service rewrite language', expectedIds: ['tmp-4'], description: 'latest should boost most recent' }, { id: 'tmp-q4', category: 'temporal', query: 'original architecture decision for the service', expectedIds: ['tmp-1'], description: 'original should boost earliest' });
94
- // =========================================================================
95
- // 5. NOISE RESISTANCE — relevant memories found despite noise
96
- // =========================================================================
97
- // Add 30 noise entries
98
- const noiseTopics = [
99
- 'breakfast meeting catering ordered sandwiches', 'office temperature thermostat adjusted',
100
- 'printer paper refill third floor supply', 'parking lot gate code changed 4521',
101
- 'fire drill scheduled next Thursday morning', 'coffee machine broken maintenance called',
102
- 'desk booking system new policy hybrid', 'meeting room projector HDMI adapter missing',
103
- 'birthday celebration for Dave next Friday', 'recycling bins moved to kitchen area',
104
- 'badge access updated for new hires', 'elevator maintenance scheduled weekend',
105
- 'lunch order from Italian place confirmed', 'desk plants watering schedule posted',
106
- 'air conditioning unit serviced last week', 'office carpet cleaning Friday evening',
107
- 'new microwave installed in kitchen', 'visitor parking available spots three',
108
- 'holiday schedule posted on intranet', 'team photo session Thursday afternoon',
109
- 'chair ergonomics assessment signup sheet', 'standing desk adjustment instructions',
110
- 'kitchen fridge cleanup policy reminder', 'window blinds replaced on south side',
111
- 'bicycle rack installed in parking garage', 'team lunch budget increased quarterly',
112
- 'noise canceling headphones approved expense', 'monitor arm request form updated',
113
- 'desk drawer key replacement procedure', 'building security hours extended',
114
- ];
115
- for (let i = 0; i < noiseTopics.length; i++) {
116
- entries.push(mem(`noise-${i}`, noiseTopics[i], { created: dateOffset(i), tags: ['topic:office'] }));
117
- }
118
- entries.push(mem('nr-1', 'The database connection pool was exhausted because max_connections was set to 10 but the application had 25 concurrent requests', {
119
- created: dateOffset(20), tags: ['topic:database', 'topic:performance'],
120
- }), mem('nr-2', 'The S3 bucket policy was misconfigured allowing public read access to customer data uploads', {
121
- created: dateOffset(21), tags: ['topic:security', 'topic:s3'],
122
- }));
123
- cases.push({ id: 'nr-q1', category: 'noise-resistance', query: 'database connection pool exhausted max connections', expectedIds: ['nr-1'], description: 'find DB issue despite 30 noise entries' }, { id: 'nr-q2', category: 'noise-resistance', query: 'S3 bucket public access security misconfiguration', expectedIds: ['nr-2'], description: 'find S3 issue despite noise' }, { id: 'nr-q3', category: 'noise-resistance', query: 'PostgreSQL migration constraint', expectedIds: ['dr-1'], description: 'find earlier DB entry despite noise' }, { id: 'nr-q4', category: 'noise-resistance', query: 'JWT token security expiration', expectedIds: ['dr-5'], description: 'find security entry despite noise' });
124
- // =========================================================================
125
- // 6. MULTI-HOP — entity chaining across sessions
126
- // =========================================================================
127
- entries.push(mem('mh-1', 'speaker:Alice works on the payment gateway integration with Stripe. She found a webhook signature validation bug.', {
128
- created: dateOffset(25), tags: ['speaker:Alice', 'topic:payments'],
129
- }), mem('mh-2', 'speaker:Bob reviewed the Stripe webhook code and confirmed the signature validation uses the wrong secret key from the test environment.', {
130
- created: dateOffset(26), tags: ['speaker:Bob', 'topic:payments'],
131
- }), mem('mh-3', 'speaker:Alice fixed the Stripe webhook by switching to the production secret key. Payment confirmations now arrive within 2 seconds.', {
132
- created: dateOffset(27), tags: ['speaker:Alice', 'topic:payments'],
133
- }), mem('mh-4', 'speaker:Carol reported that the billing dashboard shows incorrect revenue numbers because it reads from the payments_raw table instead of payments_reconciled.', {
134
- created: dateOffset(28), tags: ['speaker:Carol', 'topic:billing'],
135
- }));
136
- cases.push({ id: 'mh-q1', category: 'multi-hop', query: 'Who fixed the Stripe webhook bug and what was the root cause?', expectedIds: ['mh-1', 'mh-2', 'mh-3'], description: 'chain Alice -> Stripe -> Bob -> fix' }, { id: 'mh-q2', category: 'multi-hop', query: 'What are all the payment-related issues the team discussed?', expectedIds: ['mh-1', 'mh-2', 'mh-3', 'mh-4'], description: 'find all payment topics across speakers' });
137
- return { entries, cases };
138
- }
139
- // ---------------------------------------------------------------------------
140
- // Runner — evaluates each case against the synthetic corpus
141
- // ---------------------------------------------------------------------------
142
- export async function runFeatureEval(version) {
143
- const start = Date.now();
144
- const { entries, cases } = buildSyntheticCorpus();
145
- const caseResults = [];
146
- for (const c of cases) {
147
- let results;
148
- if (c.category === 'multi-hop') {
149
- results = multihopSearch(c.query, entries, { budget: 50000 });
150
- }
151
- else {
152
- results = search(c.query, entries, { budget: 50000 });
153
- }
154
- const returnedIds = results.map(r => r.entry.id);
155
- caseResults.push({
156
- case: c,
157
- returnedIds,
158
- mrrVal: mrr(returnedIds, c.expectedIds),
159
- r5: recallAtK(returnedIds, c.expectedIds, 5),
160
- ndcg5: ndcgAtK(returnedIds, c.expectedIds, 5),
161
- });
162
- }
163
- const categories = [...new Set(cases.map(c => c.category))];
164
- const features = categories.map(cat => {
165
- const catCases = caseResults.filter(r => r.case.category === cat);
166
- const n = catCases.length;
167
- const avgMrr = catCases.reduce((s, r) => s + r.mrrVal, 0) / n;
168
- const avgR5 = catCases.reduce((s, r) => s + r.r5, 0) / n;
169
- const avgNdcg5 = catCases.reduce((s, r) => s + r.ndcg5, 0) / n;
170
- return {
171
- category: cat,
172
- cases: n,
173
- mrr: avgMrr,
174
- recallAt5: avgR5,
175
- ndcgAt5: avgNdcg5,
176
- passed: true,
177
- };
178
- });
179
- const totalCases = caseResults.length;
180
- const overallMrr = caseResults.reduce((s, r) => s + r.mrrVal, 0) / totalCases;
181
- const overallR5 = caseResults.reduce((s, r) => s + r.r5, 0) / totalCases;
182
- const overallNdcg5 = caseResults.reduce((s, r) => s + r.ndcg5, 0) / totalCases;
183
- return {
184
- version,
185
- timestamp: new Date().toISOString(),
186
- features,
187
- overall: { mrr: overallMrr, recallAt5: overallR5, ndcgAt5: overallNdcg5 },
188
- totalCases,
189
- durationMs: Date.now() - start,
190
- };
191
- }
192
- // ---------------------------------------------------------------------------
193
- // Regression detection
194
- // ---------------------------------------------------------------------------
195
- const REGRESSION_THRESHOLD = 0.05;
196
- export function detectRegressions(baseline, current) {
197
- const regressions = [];
198
- const improvements = [];
199
- for (const feat of current.features) {
200
- const base = baseline.features[feat.category];
201
- if (!base)
202
- continue;
203
- for (const metric of ['mrr', 'recallAt5', 'ndcgAt5']) {
204
- const baseVal = base[metric];
205
- const curVal = feat[metric];
206
- const delta = curVal - baseVal;
207
- if (delta < -REGRESSION_THRESHOLD) {
208
- regressions.push({ category: feat.category, metric, baseline: baseVal, current: curVal, delta });
209
- }
210
- else if (delta > REGRESSION_THRESHOLD) {
211
- improvements.push({ category: feat.category, metric, baseline: baseVal, current: curVal, delta });
212
- }
213
- }
214
- }
215
- return {
216
- regressions,
217
- improvements,
218
- verdict: regressions.length > 0 ? 'REGRESSION' : 'PASS',
219
- };
220
- }
221
- export function resultToBaseline(result) {
222
- // SAFETY: every FeatureCategory key is populated by the loop immediately
223
- // below (one iteration per entry in result.features), before this object
224
- // is read anywhere; result.features is built from every category in
225
- // buildSyntheticCorpus's cases (runFeatureEval derives `categories` the
226
- // same way), so no FeatureCategory key is left unset.
227
- const features = {};
228
- for (const f of result.features) {
229
- features[f.category] = { mrr: f.mrr, recallAt5: f.recallAt5, ndcgAt5: f.ndcgAt5 };
230
- }
231
- return {
232
- version: result.version,
233
- timestamp: result.timestamp,
234
- features,
235
- overall: result.overall,
236
- };
237
- }
238
- // ---------------------------------------------------------------------------
239
- // Formatters
240
- // ---------------------------------------------------------------------------
241
- function pct(n) {
242
- return (n * 100).toFixed(1) + '%';
243
- }
244
- function pad(s, w) {
245
- return s.padEnd(w);
246
- }
247
- function deltaStr(d) {
248
- const sign = d >= 0 ? '+' : '';
249
- return sign + pct(d);
250
- }
251
- export function formatResult(result, baseline) {
252
- const lines = [];
253
- lines.push(`Hippo Eval v${result.version} — ${result.totalCases} queries, ${result.durationMs}ms`);
254
- lines.push('');
255
- lines.push(`${pad('Feature', 24)} | ${pad('MRR', 8)} | ${pad('R@5', 8)} | ${pad('NDCG@5', 8)} | ${pad('vs baseline', 12)} | verdict`);
256
- lines.push(`${'─'.repeat(24)}-|-${'─'.repeat(8)}-|-${'─'.repeat(8)}-|-${'─'.repeat(8)}-|-${'─'.repeat(12)}-|${'─'.repeat(8)}`);
257
- let report;
258
- if (baseline) {
259
- report = detectRegressions(baseline, result);
260
- }
261
- for (const feat of result.features) {
262
- const base = baseline?.features[feat.category];
263
- let vsBaseline = 'NEW';
264
- let verdict = 'baseline';
265
- if (base) {
266
- const delta = feat.ndcgAt5 - base.ndcgAt5;
267
- vsBaseline = deltaStr(delta);
268
- verdict = delta < -REGRESSION_THRESHOLD ? 'REGRESS' : 'PASS';
269
- }
270
- lines.push(`${pad(feat.category, 24)} | ${pad(pct(feat.mrr), 8)} | ${pad(pct(feat.recallAt5), 8)} | ${pad(pct(feat.ndcgAt5), 8)} | ${pad(vsBaseline, 12)} | ${verdict}`);
271
- }
272
- lines.push('');
273
- lines.push(`Overall: MRR ${pct(result.overall.mrr)} | R@5 ${pct(result.overall.recallAt5)} | NDCG@5 ${pct(result.overall.ndcgAt5)}`);
274
- if (report) {
275
- if (report.verdict === 'REGRESSION') {
276
- lines.push('');
277
- lines.push(`REGRESSIONS DETECTED (>${pct(REGRESSION_THRESHOLD)} drop):`);
278
- for (const r of report.regressions) {
279
- lines.push(` ${r.category}.${r.metric}: ${pct(r.baseline)} -> ${pct(r.current)} (${deltaStr(r.delta)})`);
280
- }
281
- }
282
- if (report.improvements.length > 0) {
283
- lines.push('');
284
- lines.push('Improvements:');
285
- for (const imp of report.improvements) {
286
- lines.push(` ${imp.category}.${imp.metric}: ${pct(imp.baseline)} -> ${pct(imp.current)} (${deltaStr(imp.delta)})`);
287
- }
288
- }
289
- lines.push('');
290
- lines.push(`Verdict: ${report.verdict}${report.regressions.length > 0 ? ` (${report.regressions.length} regressions)` : ''}`);
291
- }
292
- return lines.join('\n');
293
- }
294
- //# sourceMappingURL=eval-suite.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../../src/eval-suite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,KAAK,EAAoB,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,MAAM,EAA8E,MAAM,aAAa,CAAC;AACjH,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiEpD,8EAA8E;AAC9E,iDAAiD;AACjD,8EAA8E;AAE9E,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,sBAAsB,CAAC,CAAC;AAEnD,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,QAAQ,CAAC,CAAC;IAC1D,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,GAAG,CAAC,EAAU,EAAE,OAAe,EAAE,OAGtC,EAAE;IACJ,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,EAAE;QAClC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ;QACnC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;QACrB,gBAAgB,EAAE,EAAE;QACpB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;KAClC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC;IACd,IAAI,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAsB,EAAE,CAAC;IAEpC,4EAA4E;IAC5E,4CAA4C;IAC5C,4EAA4E;IAE5E,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,MAAM,EAAE,gHAAgH,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,CAAC,EAAE,CAAC,EACnL,GAAG,CAAC,MAAM,EAAE,4GAA4G,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAC/K,GAAG,CAAC,MAAM,EAAE,6GAA6G,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,EAC3K,GAAG,CAAC,MAAM,EAAE,oGAAoG,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,EACrK,GAAG,CAAC,MAAM,EAAE,sGAAsG,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,CAAC,EAAE,CAAC,EACzK,GAAG,CAAC,MAAM,EAAE,oGAAoG,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,CAAC,EAAE,CAAC,EACvK,GAAG,CAAC,MAAM,EAAE,iGAAiG,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,EACnK,GAAG,CAAC,MAAM,EAAE,0FAA0F,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CACzJ,CAAC;IAEF,KAAK,CAAC,IAAI,CACR,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,0CAA0C,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,qCAAqC,EAAE,EACxK,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,qCAAqC,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,EAC9J,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,iCAAiC,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,4BAA4B,EAAE,EACtJ,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,iCAAiC,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE,EAChJ,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,+BAA+B,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE,EAC9I,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,4BAA4B,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,mBAAmB,EAAE,EACxI,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,oCAAoC,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,EAC/I,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,iCAAiC,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,iBAAiB,EAAE,CAC5I,CAAC;IAEF,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAE5E,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,UAAU,EAAE,8OAA8O,EAAE;QAC9P,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,sBAAsB,EAAE,mBAAmB,CAAC;KAC9F,CAAC,EACF,GAAG,CAAC,UAAU,EAAE,qHAAqH,EAAE;QACrI,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,sBAAsB,EAAE,WAAW,CAAC;QAC5G,cAAc,EAAE,UAAU;KAC3B,CAAC,EACF,GAAG,CAAC,UAAU,EAAE,iPAAiP,EAAE;QACjQ,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,aAAa,EAAE,kBAAkB,EAAE,mBAAmB,CAAC;KACxF,CAAC,EACF,GAAG,CAAC,UAAU,EAAE,+JAA+J,EAAE;QAC/K,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,aAAa,EAAE,kBAAkB,EAAE,WAAW,CAAC;QACtG,cAAc,EAAE,UAAU;KAC3B,CAAC,EACF,GAAG,CAAC,UAAU,EAAE,mKAAmK,EAAE;QACnL,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,YAAY,EAAE,mBAAmB,CAAC;KACpF,CAAC,EACF,GAAG,CAAC,UAAU,EAAE,yJAAyJ,EAAE;QACzK,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,YAAY,EAAE,WAAW,CAAC;QAClG,cAAc,EAAE,UAAU;KAC3B,CAAC,CACH,CAAC;IAEF,KAAK,CAAC,IAAI,CACR,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,KAAK,EAAE,wCAAwC,EAAE,WAAW,EAAE,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,kEAAkE,EAAE,EAC/M,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,KAAK,EAAE,oCAAoC,EAAE,WAAW,EAAE,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,wCAAwC,EAAE,EACjL,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,KAAK,EAAE,iCAAiC,EAAE,WAAW,EAAE,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,CACvK,CAAC;IAEF,4EAA4E;IAC5E,4DAA4D;IAC5D,4EAA4E;IAE5E,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,aAAa,EAAE,0GAA0G,EAAE;QAC7H,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,uBAAuB,EAAE,WAAW,CAAC;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC,EACF,GAAG,CAAC,aAAa,EAAE,yGAAyG,EAAE;QAC5H,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,uBAAuB,EAAE,WAAW,CAAC;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC,EACF,GAAG,CAAC,aAAa,EAAE,iHAAiH,EAAE;QACpI,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,uBAAuB,EAAE,WAAW,CAAC;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC,EACF,GAAG,CAAC,eAAe,EAAE,oJAAoJ,EAAE;QACzK,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,uBAAuB,CAAC;QAC/E,SAAS,EAAE,CAAC,EAAE,aAAa,EAAE,SAAS;KACvC,CAAC,CACH,CAAC;IACF,0BAA0B;IAC1B,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAE,CAAC,EAAE,CAAC;QACzJ,KAAK,CAAC,aAAa,GAAG,eAAe,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,kCAAkC,EAAE,WAAW,EAAE,CAAC,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,EAC7N,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,oCAAoC,EAAE,WAAW,EAAE,CAAC,aAAa,EAAE,aAAa,EAAE,eAAe,CAAC,EAAE,WAAW,EAAE,yDAAyD,EAAE,CAC/N,CAAC;IAEF,4EAA4E;IAC5E,0DAA0D;IAC1D,4EAA4E;IAE5E,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,OAAO,EAAE,wDAAwD,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAClI,GAAG,CAAC,OAAO,EAAE,oEAAoE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAC9I,GAAG,CAAC,OAAO,EAAE,mEAAmE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAC7I,GAAG,CAAC,OAAO,EAAE,sEAAsE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAChJ,CAAC;IAEF,KAAK,CAAC,IAAI,CACR,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,8DAA8D,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,sCAAsC,EAAE,EAC1L,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,oDAAoD,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,wCAAwC,EAAE,EAClL,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,+CAA+C,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,iCAAiC,EAAE,EACtK,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,gDAAgD,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,CACvK,CAAC;IAEF,4EAA4E;IAC5E,8DAA8D;IAC9D,4EAA4E;IAE5E,uBAAuB;IACvB,MAAM,WAAW,GAAG;QAClB,+CAA+C,EAAE,wCAAwC;QACzF,yCAAyC,EAAE,oCAAoC;QAC/E,4CAA4C,EAAE,0CAA0C;QACxF,uCAAuC,EAAE,6CAA6C;QACtF,2CAA2C,EAAE,sCAAsC;QACnF,oCAAoC,EAAE,wCAAwC;QAC9E,0CAA0C,EAAE,sCAAsC;QAClF,0CAA0C,EAAE,uCAAuC;QACnF,oCAAoC,EAAE,uCAAuC;QAC7E,qCAAqC,EAAE,uCAAuC;QAC9E,0CAA0C,EAAE,uCAAuC;QACnF,wCAAwC,EAAE,sCAAsC;QAChF,0CAA0C,EAAE,uCAAuC;QACnF,6CAA6C,EAAE,kCAAkC;QACjF,uCAAuC,EAAE,kCAAkC;KAC5E,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACtG,CAAC;IAED,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,MAAM,EAAE,iIAAiI,EAAE;QAC7I,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,EAAE,mBAAmB,CAAC;KACvE,CAAC,EACF,GAAG,CAAC,MAAM,EAAE,6FAA6F,EAAE;QACzG,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,EAAE,UAAU,CAAC;KAC9D,CAAC,CACH,CAAC;IAEF,KAAK,CAAC,IAAI,CACR,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,KAAK,EAAE,oDAAoD,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,wCAAwC,EAAE,EACxL,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,KAAK,EAAE,mDAAmD,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,6BAA6B,EAAE,EAC5K,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,KAAK,EAAE,iCAAiC,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,qCAAqC,EAAE,EAClK,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,KAAK,EAAE,+BAA+B,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,mCAAmC,EAAE,CAC/J,CAAC;IAEF,4EAA4E;IAC5E,iDAAiD;IACjD,4EAA4E;IAE5E,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,MAAM,EAAE,mHAAmH,EAAE;QAC/H,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,gBAAgB,CAAC;KACnE,CAAC,EACF,GAAG,CAAC,MAAM,EAAE,0IAA0I,EAAE;QACtJ,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,aAAa,EAAE,gBAAgB,CAAC;KACjE,CAAC,EACF,GAAG,CAAC,MAAM,EAAE,sIAAsI,EAAE;QAClJ,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,gBAAgB,CAAC;KACnE,CAAC,EACF,GAAG,CAAC,MAAM,EAAE,gKAAgK,EAAE;QAC5K,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC;KAClE,CAAC,CACH,CAAC;IAEF,KAAK,CAAC,IAAI,CACR,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,+DAA+D,EAAE,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,qCAAqC,EAAE,EACzM,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,6DAA6D,EAAE,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,yCAAyC,EAAE,CACpN,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,oBAAoB,EAAE,CAAC;IAElD,MAAM,WAAW,GAAuG,EAAE,CAAC;IAE3H,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YAC/B,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAEjD,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,CAAC;YACP,WAAW;YACX,MAAM,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,CAAC;YACvC,EAAE,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YAC5C,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAoB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QACrD,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,CAAC;QAClE,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/D,OAAO;YACL,QAAQ,EAAE,GAAG;YACb,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,MAAM;YACX,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC;IAC9E,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC;IACzE,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC;IAE/E,OAAO;QACL,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ;QACR,OAAO,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE;QACzE,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC/B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,MAAM,UAAU,iBAAiB,CAAC,QAAsB,EAAE,OAAwB;IAChF,MAAM,WAAW,GAAoC,EAAE,CAAC;IACxD,MAAM,YAAY,GAAqC,EAAE,CAAC;IAE1D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,KAAK,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAU,EAAE,CAAC;YAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,MAAM,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC;YAE/B,IAAI,KAAK,GAAG,CAAC,oBAAoB,EAAE,CAAC;gBAClC,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACnG,CAAC;iBAAM,IAAI,KAAK,GAAG,oBAAoB,EAAE,CAAC;gBACxC,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,WAAW;QACX,YAAY;QACZ,OAAO,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAuB;IACtD,yEAAyE;IACzE,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,sDAAsD;IACtD,MAAM,QAAQ,GAA6B,EAA8B,CAAC;IAC1E,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAChC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IACpF,CAAC;IACD,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ;QACR,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS;IAC/B,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/B,OAAO,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAuB,EAAE,QAAuB;IAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,OAAO,MAAM,MAAM,CAAC,UAAU,aAAa,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;IACnG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,aAAa,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC;IACtI,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAE/H,IAAI,MAAoC,CAAC;IACzC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,OAAO,GAAG,UAAU,CAAC;QACzB,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC1C,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC7B,OAAO,GAAG,KAAK,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/D,CAAC;QACD,KAAK,CAAC,IAAI,CACR,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,OAAO,EAAE,CAC7J,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAErI,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,MAAM,CAAC,OAAO,KAAK,YAAY,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YACzE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACtH,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,WAAW,CAAC,MAAM,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChI,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
package/dist/src/eval.js DELETED
@@ -1,187 +0,0 @@
1
- /**
2
- * Recall eval harness.
3
- *
4
- * Given a corpus of (query, expected_memory_ids) cases, run recall against
5
- * the store and report ranking-quality metrics: MRR, Recall@K, NDCG@K.
6
- *
7
- * The goal is to make recall quality measurable so MMR lambda, embedding
8
- * weights, and future scoring tweaks can be tuned against evidence instead
9
- * of intuition.
10
- */
11
- import { hybridSearch } from './search.js';
12
- import { searchBothHybrid } from './shared.js';
13
- import { isInitialized } from './store.js';
14
- // ---------------------------------------------------------------------------
15
- // Metrics — pure functions. K is inclusive of position K.
16
- // ---------------------------------------------------------------------------
17
- /** Mean Reciprocal Rank for a single ranking given expected ids. */
18
- export function mrr(returned, expected) {
19
- if (expected.length === 0)
20
- return 0;
21
- const expectedSet = new Set(expected);
22
- for (let i = 0; i < returned.length; i++) {
23
- if (expectedSet.has(returned[i]))
24
- return 1 / (i + 1);
25
- }
26
- return 0;
27
- }
28
- /** Recall@K — fraction of expected items found in the top-K. */
29
- export function recallAtK(returned, expected, k) {
30
- if (expected.length === 0)
31
- return 0;
32
- const expectedSet = new Set(expected);
33
- const topK = returned.slice(0, k);
34
- let hits = 0;
35
- for (const id of topK) {
36
- if (expectedSet.has(id))
37
- hits++;
38
- }
39
- return hits / expected.length;
40
- }
41
- /**
42
- * Normalized Discounted Cumulative Gain at K with binary relevance.
43
- * gain_i = 1 if returned[i] ∈ expected else 0. discount = log2(i + 2).
44
- */
45
- export function ndcgAtK(returned, expected, k) {
46
- if (expected.length === 0)
47
- return 0;
48
- const expectedSet = new Set(expected);
49
- let dcg = 0;
50
- for (let i = 0; i < Math.min(k, returned.length); i++) {
51
- if (expectedSet.has(returned[i])) {
52
- dcg += 1 / Math.log2(i + 2);
53
- }
54
- }
55
- // Ideal DCG: all relevant items at top positions.
56
- const idealHits = Math.min(k, expected.length);
57
- let idcg = 0;
58
- for (let i = 0; i < idealHits; i++) {
59
- idcg += 1 / Math.log2(i + 2);
60
- }
61
- return idcg === 0 ? 0 : dcg / idcg;
62
- }
63
- // ---------------------------------------------------------------------------
64
- // Runner
65
- // ---------------------------------------------------------------------------
66
- export async function runEval(cases, entries, options = {}) {
67
- const budget = options.budget ?? 100_000; // generous so metrics aren't truncated
68
- const start = Date.now();
69
- const results = [];
70
- const useBothStores = Boolean(options.globalRoot && options.hippoRoot && isInitialized(options.globalRoot));
71
- for (const c of cases) {
72
- const ranked = useBothStores
73
- ? await searchBothHybrid(c.query, options.hippoRoot, options.globalRoot, {
74
- budget,
75
- now: options.now,
76
- embeddingWeight: options.embeddingWeight,
77
- mmr: options.mmr,
78
- mmrLambda: options.mmrLambda,
79
- localBump: options.localBump,
80
- })
81
- : await hybridSearch(c.query, entries, {
82
- budget,
83
- now: options.now,
84
- hippoRoot: options.hippoRoot,
85
- embeddingWeight: options.embeddingWeight,
86
- mmr: options.mmr,
87
- mmrLambda: options.mmrLambda,
88
- });
89
- const returnedIds = ranked.map((r) => r.entry.id);
90
- results.push({
91
- case: c,
92
- returnedIds,
93
- mrr: mrr(returnedIds, c.expectedIds),
94
- recallAt5: recallAtK(returnedIds, c.expectedIds, 5),
95
- recallAt10: recallAtK(returnedIds, c.expectedIds, 10),
96
- ndcgAt10: ndcgAtK(returnedIds, c.expectedIds, 10),
97
- });
98
- }
99
- const n = Math.max(1, results.length);
100
- const meanMrr = results.reduce((s, r) => s + r.mrr, 0) / n;
101
- const meanRecallAt5 = results.reduce((s, r) => s + r.recallAt5, 0) / n;
102
- const meanRecallAt10 = results.reduce((s, r) => s + r.recallAt10, 0) / n;
103
- const meanNdcgAt10 = results.reduce((s, r) => s + r.ndcgAt10, 0) / n;
104
- return {
105
- cases: results,
106
- meanMrr,
107
- meanRecallAt5,
108
- meanRecallAt10,
109
- meanNdcgAt10,
110
- durationMs: Date.now() - start,
111
- };
112
- }
113
- /**
114
- * Compute pairwise deltas between a baseline and a current eval summary.
115
- * Cases are matched by EvalCase.id; any mismatch is surfaced in the onlyIn
116
- * arrays so the caller knows the corpora diverged.
117
- */
118
- export function compareSummaries(baseline, current) {
119
- const aggregate = {
120
- mrr: current.meanMrr - baseline.meanMrr,
121
- recallAt5: current.meanRecallAt5 - baseline.meanRecallAt5,
122
- recallAt10: current.meanRecallAt10 - baseline.meanRecallAt10,
123
- ndcgAt10: current.meanNdcgAt10 - baseline.meanNdcgAt10,
124
- };
125
- const baseById = new Map(baseline.cases.map((c) => [c.case.id, c]));
126
- const curById = new Map(current.cases.map((c) => [c.case.id, c]));
127
- const improved = [];
128
- const regressed = [];
129
- let unchanged = 0;
130
- for (const [id, cur] of curById) {
131
- const base = baseById.get(id);
132
- if (!base)
133
- continue;
134
- const delta = {
135
- id,
136
- query: cur.case.query,
137
- mrrBefore: base.mrr,
138
- mrrAfter: cur.mrr,
139
- r10Before: base.recallAt10,
140
- r10After: cur.recallAt10,
141
- ndcgBefore: base.ndcgAt10,
142
- ndcgAfter: cur.ndcgAt10,
143
- };
144
- const ndcgDelta = delta.ndcgAfter - delta.ndcgBefore;
145
- if (ndcgDelta > 1e-6)
146
- improved.push(delta);
147
- else if (ndcgDelta < -1e-6)
148
- regressed.push(delta);
149
- else
150
- unchanged++;
151
- }
152
- return {
153
- aggregate,
154
- improved: improved.sort((a, b) => (b.ndcgAfter - b.ndcgBefore) - (a.ndcgAfter - a.ndcgBefore)),
155
- regressed: regressed.sort((a, b) => (a.ndcgAfter - a.ndcgBefore) - (b.ndcgAfter - b.ndcgBefore)),
156
- unchanged,
157
- onlyInBaseline: [...baseById.keys()].filter((id) => !curById.has(id)),
158
- onlyInCurrent: [...curById.keys()].filter((id) => !baseById.has(id)),
159
- };
160
- }
161
- // ---------------------------------------------------------------------------
162
- // Bootstrap — generate a synthetic corpus from current memories
163
- // ---------------------------------------------------------------------------
164
- /**
165
- * For each memory, take its first 8 content words as a trivial query and
166
- * expect that memory back. Useful as a smoke test: if recall can't find a
167
- * memory by its own opening words, something is broken.
168
- */
169
- export function bootstrapCorpus(entries, maxCases = 50) {
170
- const cases = [];
171
- for (const e of entries) {
172
- if (cases.length >= maxCases)
173
- break;
174
- const words = e.content.trim().split(/\s+/).filter((w) => w.length > 2);
175
- if (words.length < 3)
176
- continue;
177
- const query = words.slice(0, 8).join(' ');
178
- cases.push({
179
- id: `bootstrap_${e.id}`,
180
- query,
181
- expectedIds: [e.id],
182
- description: `trivial self-query on memory ${e.id}`,
183
- });
184
- }
185
- return cases;
186
- }
187
- //# sourceMappingURL=eval.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAuD3C,8EAA8E;AAC9E,0DAA0D;AAC1D,8EAA8E;AAE9E,oEAAoE;AACpE,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAE,QAAkB;IACxD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,SAAS,CAAC,QAAkB,EAAE,QAAkB,EAAE,CAAS;IACzE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,IAAI,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,QAAkB,EAAE,QAAkB,EAAE,CAAS;IACvE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,kDAAkD;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;AACrC,CAAC;AAED,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAiB,EACjB,OAAsB,EACtB,UAA0B,EAAE;IAE5B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAG,uCAAuC;IACnF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,MAAM,aAAa,GAAG,OAAO,CAC3B,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAC7E,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,aAAa;YAC1B,CAAC,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,SAAU,EAAE,OAAO,CAAC,UAAW,EAAE;gBACvE,MAAM;gBACN,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC;YACJ,CAAC,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE;gBACnC,MAAM;gBACN,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAC;QACP,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,CAAC;YACP,WAAW;YACX,GAAG,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,CAAC;YACpC,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACnD,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACrD,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IACzE,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAErE,OAAO;QACL,KAAK,EAAE,OAAO;QACd,OAAO;QACP,aAAa;QACb,cAAc;QACd,YAAY;QACZ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC/B,CAAC;AACJ,CAAC;AAiCD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAqB,EAAE,OAAoB;IAC1E,MAAM,SAAS,GAAc;QAC3B,GAAG,EAAE,OAAO,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO;QACvC,SAAS,EAAE,OAAO,CAAC,aAAa,GAAG,QAAQ,CAAC,aAAa;QACzD,UAAU,EAAE,OAAO,CAAC,cAAc,GAAG,QAAQ,CAAC,cAAc;QAC5D,QAAQ,EAAE,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY;KACvD,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,QAAQ,GAAgB,EAAE,CAAC;IACjC,MAAM,SAAS,GAAgB,EAAE,CAAC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,OAAO,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAc;YACvB,EAAE;YACF,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,QAAQ,EAAE,GAAG,CAAC,GAAG;YACjB,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,QAAQ,EAAE,GAAG,CAAC,UAAU;YACxB,UAAU,EAAE,IAAI,CAAC,QAAQ;YACzB,SAAS,EAAE,GAAG,CAAC,QAAQ;SACxB,CAAC;QACF,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC;QACrD,IAAI,SAAS,GAAG,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACtC,IAAI,SAAS,GAAG,CAAC,IAAI;YAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;YAC7C,SAAS,EAAE,CAAC;IACnB,CAAC;IAED,OAAO;QACL,SAAS;QACT,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAC9F,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAChG,SAAS;QACT,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,aAAa,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,gEAAgE;AAChE,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAAsB,EAAE,QAAQ,GAAG,EAAE;IACnE,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;YAAE,MAAM;QACpC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC;YACT,EAAE,EAAE,aAAa,CAAC,CAAC,EAAE,EAAE;YACvB,KAAK;YACL,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,WAAW,EAAE,gCAAgC,CAAC,CAAC,EAAE,EAAE;SACpD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}